{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First ensemble with NSL-KDD\n",
    "# Parameters\n",
    "# Few parameters are not fully implemented yet\n",
    "\n",
    "#----------------------------------------------\n",
    "# 0 for not using it as base learner\n",
    "# 1 for using it as base learner\n",
    "# not implemented but in the code in someparts\n",
    "use_model_ada = 1 \n",
    "use_model_dnn = 1 \n",
    "use_model_mlp = 1 \n",
    "use_model_lgbm = 1 \n",
    "use_model_rf = 1 \n",
    "use_model_svm = 1\n",
    "use_model_knn = 1 \n",
    "#----------------------------------------------\n",
    "# 0 for training the model\n",
    "# 1 for using the saved version of the model\n",
    "\n",
    "# load_model_ada = 0 \n",
    "# load_model_dnn = 0 \n",
    "# load_model_mlp = 0 \n",
    "# load_model_lgbm = 0 \n",
    "# load_model_rf = 0 \n",
    "# load_model_svm = 0\n",
    "# load_model_knn = 0 \n",
    "#----------------------------------------------\n",
    "# not implemented but in the code in someparts\n",
    "load_model_ada = 1\n",
    "load_model_dnn = 1 \n",
    "load_model_mlp = 1 \n",
    "load_model_lgbm = 1 \n",
    "load_model_rf = 1                               \n",
    "load_model_svm = 1\n",
    "load_model_knn = 1 \n",
    "#----------------------------------------------\n",
    "\n",
    "# Implemented\n",
    "#----------------------------------------------\n",
    "# feature_selection_bit = 0 # OFF\n",
    "feature_selection_bit = 0 # On\n",
    "# pick_prob = 1 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "pick_prob = 1\n",
    "generate_feature_importance = 0 # Generate Shap graphs\n",
    "\n",
    "\n",
    "column_features = [\n",
    "                    'dnn',\n",
    "                   'rf',\n",
    "                   'lgbm',\n",
    "                   'ada',\n",
    "                #    'knn',\n",
    "                #    'mlp',\n",
    "                   'svm',\n",
    "                   'cat',\n",
    "                   'xgb',\n",
    "                   'lr',\n",
    "                   'dt',\n",
    "                   'label']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify the name of the output text file\n",
    "if feature_selection_bit == 0:\n",
    "\n",
    "    if pick_prob == 0:\n",
    "        output_file_name = \"ensemble_level_01_all_features_classes.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_all_features_classes--', file = f)\n",
    "\n",
    "    elif pick_prob == 1:\n",
    "        output_file_name = \"ensemble_level_01_all_features_probabilites.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_all_features_probabilites--', file = f)\n",
    "\n",
    "elif feature_selection_bit == 1:\n",
    "    if pick_prob == 0:\n",
    "        output_file_name = \"ensemble_level_01_feature_selection_classes.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_feature_selection_classes--', file = f)\n",
    "    elif pick_prob == 1:\n",
    "        output_file_name = \"ensemble_level_01_feature_selection_probabilites.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_feature_selection_probabilites--', file = f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding: utf-8\n",
    "\n",
    "# In[1]:\n",
    "# importing required libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle # saving and loading trained model\n",
    "from os import path\n",
    "\n",
    "\n",
    "# importing required libraries for normalizing data\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import (StandardScaler, OrdinalEncoder,LabelEncoder, MinMaxScaler, OneHotEncoder)\n",
    "from sklearn.preprocessing import Normalizer, MaxAbsScaler , RobustScaler, PowerTransformer\n",
    "\n",
    "# importing library for plotting\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import accuracy_score # for calculating accuracy of model\n",
    "from sklearn.model_selection import train_test_split # for splitting the dataset for training and testing\n",
    "from sklearn.metrics import classification_report # for generating a classification report of model\n",
    "\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "from keras.layers import Dense # importing dense layer\n",
    "\n",
    "from keras.layers import Input\n",
    "from keras.models import Model\n",
    "# representation of model layers\n",
    "#from keras.utils import plot_model\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import shap\n",
    "import time\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "import joblib\n",
    "from sklearn.model_selection import train_test_split\n",
    "import sklearn\n",
    "from tabulate import tabulate\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding: utf-8\n",
    "\n",
    "# In[1]:\n",
    "# importing required libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle # saving and loading trained model\n",
    "from os import path\n",
    "\n",
    "\n",
    "# importing required libraries for normalizing data\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import (StandardScaler, OrdinalEncoder,LabelEncoder, MinMaxScaler, OneHotEncoder)\n",
    "from sklearn.preprocessing import Normalizer, MaxAbsScaler , RobustScaler, PowerTransformer\n",
    "\n",
    "# importing library for plotting\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import accuracy_score # for calculating accuracy of model\n",
    "from sklearn.model_selection import train_test_split # for splitting the dataset for training and testing\n",
    "from sklearn.metrics import classification_report # for generating a classification report of model\n",
    "\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "from keras.layers import Dense # importing dense layer\n",
    "\n",
    "from keras.layers import Input\n",
    "from keras.models import Model\n",
    "# representation of model layers\n",
    "#from keras.utils import plot_model\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import shap\n",
    "\n",
    "import time\n",
    "start_program = time.time()\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "import joblib\n",
    "from sklearn.model_selection import train_test_split\n",
    "import sklearn\n",
    "from tabulate import tabulate\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def confusion_metrics (name_model,predictions,true_labels):\n",
    "\n",
    "    name = name_model\n",
    "    pred_label = predictions\n",
    "    y_test_01 = true_labels \n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(name, file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    Acc = accuracy_score(y_test_01, pred_label)\n",
    "    Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "\n",
    "    # voting_acc_01 = Acc\n",
    "    # voting_pre_01 = Precision\n",
    "    # weighed_avg_rec_01 = Recall\n",
    "    # weighed_avg_f1_01 = F1\n",
    "    # weighed_avg_bacc_01 = BACC\n",
    "    # weighed_avg_mcc_01 = MCC\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n",
    "    return Acc, Precision, Recall, F1, BACC, MCC\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df_level_00_1=pd.read_csv('base_models_prob_feature_selection.csv')\n",
    "df_level_00_0=pd.read_csv('base_models_class_feature_selection.csv')\n",
    "\n",
    "\n",
    "df_level_00_1\n",
    "y1 = df_level_00_1.pop('label')\n",
    "X1 = df_level_00_1\n",
    "df_level_00_1 = X1.assign(label = y1)\n",
    "y0 = df_level_00_0.pop('label')\n",
    "X0 = df_level_00_0\n",
    "df_level_00_0 = X0.assign(label = y0)\n",
    "\n",
    "if feature_selection_bit == 1:\n",
    "\n",
    "    from sklearn.feature_selection import mutual_info_classif\n",
    "    %matplotlib inline\n",
    "\n",
    "    # Compute information gain using mutual information\n",
    "    importances0 = mutual_info_classif(X0, y0)\n",
    "    importances1 = mutual_info_classif(X1, y1)\n",
    "\n",
    "\n",
    "    feat_importances0 = pd.Series(importances0, df_level_00_0.columns[0:len(df_level_00_0.columns)-1])\n",
    "    feat_importances1= pd.Series(importances1, df_level_00_1.columns[0:len(df_level_00_1.columns)-1])\n",
    "\n",
    "    # feat_importances.plot(kind='barh', color = 'teal')\n",
    "        \n",
    "    feat_importances_sorted0 = feat_importances0.sort_values( ascending=False)\n",
    "    feat_importances_sorted1 = feat_importances1.sort_values( ascending=False)\n",
    "\n",
    "\n",
    "    # Print or use the sorted DataFrame\n",
    "    print(feat_importances_sorted0)\n",
    "    print(feat_importances_sorted1)\n",
    "\n",
    "    # feat_importances_sorted.plot(kind='barh', color = 'teal')\n",
    "    # feat_importances_sorted\n",
    "    top_features0 = feat_importances_sorted0.nlargest(5)\n",
    "    top_features1 = feat_importances_sorted1.nlargest(5)\n",
    "\n",
    "    top_feature_names0 = top_features0.index.tolist()\n",
    "    top_feature_names1 = top_features1.index.tolist()\n",
    "\n",
    "\n",
    "    print(\"Top 5 feature names:\")\n",
    "    print(top_feature_names0)\n",
    "    print(top_feature_names1)\n",
    "\n",
    "    column_features0 = top_feature_names0\n",
    "    column_features1 = top_feature_names1\n",
    "\n",
    "    # df_level_00_0 = df_level_00_0[column_features0]\n",
    "    # df_level_00_1 = df_level_00_1[column_features1]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Assuming df is your DataFrame\n",
    "# if feature_selection_bit == 1:\n",
    "#     df_level_00_1=pd.read_csv('base_models_prob_feature_selection.csv',names=column_features)\n",
    "#     df_level_00_0=pd.read_csv('base_models_class_feature_selection.csv',names=column_features)\n",
    "\n",
    "# if feature_selection_bit == 0:\n",
    "\n",
    "#     df_level_00_1=pd.read_csv('base_models_prob_feature_selection.csv')\n",
    "#     df_level_00_0=pd.read_csv('base_models_class_feature_selection.csv')\n",
    "\n",
    "df_level_00_1=pd.read_csv('base_models_prob_feature_selection.csv')\n",
    "df_level_00_0=pd.read_csv('base_models_class_feature_selection.csv')\n",
    "# df_level_00_1 = df_level_00_1.sample(frac = 0.0001)\n",
    "# df_level_00_0 = df_level_00_0.sample(frac = 0.0001)\n",
    "if feature_selection_bit == 1:\n",
    "    df_level_00_0 = df_level_00_0[column_features0]\n",
    "    df_level_00_1 = df_level_00_1[column_features1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>ada</th>\n",
       "      <th>svm</th>\n",
       "      <th>cat</th>\n",
       "      <th>xgb</th>\n",
       "      <th>lr</th>\n",
       "      <th>dt</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1736432</th>\n",
       "      <td>0.994511</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999974</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999296</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.971514</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>703063</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.996837</td>\n",
       "      <td>0.999997</td>\n",
       "      <td>0.603321</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999480</td>\n",
       "      <td>0.822928</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1227234</th>\n",
       "      <td>0.994511</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999974</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999296</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.971514</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135368</th>\n",
       "      <td>0.999995</td>\n",
       "      <td>0.999958</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999446</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.811936</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>525413</th>\n",
       "      <td>0.751246</td>\n",
       "      <td>0.988915</td>\n",
       "      <td>0.995684</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.995473</td>\n",
       "      <td>0.967483</td>\n",
       "      <td>0.991587</td>\n",
       "      <td>0.995678</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1656809</th>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.999759</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999282</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.905582</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1337871</th>\n",
       "      <td>0.986539</td>\n",
       "      <td>0.999150</td>\n",
       "      <td>0.999989</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.998506</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.689570</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>765193</th>\n",
       "      <td>0.740257</td>\n",
       "      <td>0.988649</td>\n",
       "      <td>0.992978</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.992696</td>\n",
       "      <td>0.965080</td>\n",
       "      <td>0.989074</td>\n",
       "      <td>0.992987</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>825194</th>\n",
       "      <td>0.994512</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999983</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999321</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.971515</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1330447</th>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.999759</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999282</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.914439</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>180 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              dnn        rf      lgbm       ada       svm       cat       xgb  \\\n",
       "1736432  0.994511  0.995236  0.999974  0.430217  0.333333  0.999296  0.971597   \n",
       "703063   1.000000  0.996837  0.999997  0.603321  0.333333  0.999480  0.822928   \n",
       "1227234  0.994511  0.995236  0.999974  0.430217  0.333333  0.999296  0.971597   \n",
       "135368   0.999995  0.999958  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
       "525413   0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "1656809  0.999999  0.999759  0.999999  0.559890  0.333333  0.999282  0.974612   \n",
       "1337871  0.986539  0.999150  0.999989  0.559890  0.333333  0.998506  0.974612   \n",
       "765193   0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
       "825194   0.994512  0.995236  0.999983  0.430217  0.333333  0.999321  0.971597   \n",
       "1330447  0.999999  0.999759  0.999999  0.559890  0.333333  0.999282  0.974612   \n",
       "\n",
       "               lr        dt  label  \n",
       "1736432  0.971514  1.000000    0.0  \n",
       "703063   1.000000  1.000000    1.0  \n",
       "1227234  0.971514  1.000000    0.0  \n",
       "135368   0.811936  1.000000    1.0  \n",
       "525413   0.991587  0.995678    2.0  \n",
       "...           ...       ...    ...  \n",
       "1656809  0.905582  1.000000    1.0  \n",
       "1337871  0.689570  1.000000    1.0  \n",
       "765193   0.989074  0.992987    2.0  \n",
       "825194   0.971515  1.000000    0.0  \n",
       "1330447  0.914439  1.000000    1.0  \n",
       "\n",
       "[180 rows x 10 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>ada</th>\n",
       "      <th>svm</th>\n",
       "      <th>cat</th>\n",
       "      <th>xgb</th>\n",
       "      <th>lr</th>\n",
       "      <th>dt</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1432923</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1388026</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1446448</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1060754</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1449348</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>169752</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1518191</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1578643</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1292927</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>913272</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>180 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         dnn   rf  lgbm  ada  svm  cat  xgb   lr   dt  label\n",
       "1432923  1.0  1.0   1.0  1.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "1388026  2.0  2.0   2.0  2.0  2.0  2.0  2.0  2.0  2.0    2.0\n",
       "1446448  2.0  2.0   2.0  2.0  2.0  2.0  2.0  2.0  2.0    2.0\n",
       "1060754  2.0  2.0   2.0  2.0  2.0  2.0  2.0  2.0  2.0    2.0\n",
       "1449348  0.0  0.0   0.0  1.0  0.0  0.0  0.0  0.0  0.0    0.0\n",
       "...      ...  ...   ...  ...  ...  ...  ...  ...  ...    ...\n",
       "169752   1.0  1.0   1.0  1.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "1518191  0.0  0.0   0.0  1.0  0.0  0.0  0.0  0.0  0.0    0.0\n",
       "1578643  1.0  1.0   1.0  1.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "1292927  1.0  1.0   1.0  1.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "913272   1.0  1.0   1.0  0.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "\n",
       "[180 rows x 10 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0          1.0\n",
       "1          0.0\n",
       "2          0.0\n",
       "3          1.0\n",
       "4          2.0\n",
       "          ... \n",
       "1799995    1.0\n",
       "1799996    2.0\n",
       "1799997    1.0\n",
       "1799998    1.0\n",
       "1799999    1.0\n",
       "Name: label, Length: 1800000, dtype: float64"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "if pick_prob == 1:\n",
    "    df_level_01 = df_level_00_1\n",
    "else: \n",
    "    df_level_01 = df_level_00_0\n",
    "\n",
    "df_level_01 = df_level_01.assign(label = y1)\n",
    "\n",
    "y_01 = df_level_01.pop('label')\n",
    "X_01 = df_level_01\n",
    "df_level_01 = df_level_01.assign(label = y_01)\n",
    "\n",
    "\n",
    "split = 0.7\n",
    "X_train_01,X_test_01, y_train_01, y_test_01 = sklearn.model_selection.train_test_split(X_01, y_01, train_size=split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# df_level_02 = pd.read_csv('base_models_class_feature_selection.csv')\n",
    "\n",
    "# df_level_02\n",
    "\n",
    "# y_02 = df_level_02.pop('label')\n",
    "# X_02 = df_level_02\n",
    "# df_level_02 = df_level_02.assign(label = y_01)\n",
    "\n",
    "\n",
    "# split = 0.7\n",
    "# X_train_02,X_test_02, y_train_02, y_test_02 = sklearn.model_selection.train_test_split(X_02, y_02, train_size=split)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training the stronger model - STACK level 01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "#----------------------------------------------------------------\n",
    "with open(output_file_name, \"a\") as f: print('Stack model - Strong learner - level 01', file = f)\n",
    "with open(output_file_name, \"a\") as f: print('-------------------------------------------------------', file = f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>ada</th>\n",
       "      <th>svm</th>\n",
       "      <th>cat</th>\n",
       "      <th>xgb</th>\n",
       "      <th>lr</th>\n",
       "      <th>dt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>567509</th>\n",
       "      <td>0.994514</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999923</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999321</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.971516</td>\n",
       "      <td>0.999925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34764</th>\n",
       "      <td>0.740257</td>\n",
       "      <td>0.988649</td>\n",
       "      <td>0.992978</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.992696</td>\n",
       "      <td>0.965080</td>\n",
       "      <td>0.989074</td>\n",
       "      <td>0.992987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>991802</th>\n",
       "      <td>0.751246</td>\n",
       "      <td>0.988915</td>\n",
       "      <td>0.995684</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.995473</td>\n",
       "      <td>0.967483</td>\n",
       "      <td>0.991587</td>\n",
       "      <td>0.995678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>647470</th>\n",
       "      <td>0.991945</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999790</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999296</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.970398</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1260575</th>\n",
       "      <td>0.740257</td>\n",
       "      <td>0.988649</td>\n",
       "      <td>0.992978</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.992696</td>\n",
       "      <td>0.965080</td>\n",
       "      <td>0.989074</td>\n",
       "      <td>0.992987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1141227</th>\n",
       "      <td>0.740257</td>\n",
       "      <td>0.988649</td>\n",
       "      <td>0.992978</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.992696</td>\n",
       "      <td>0.965080</td>\n",
       "      <td>0.989074</td>\n",
       "      <td>0.992987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1774849</th>\n",
       "      <td>0.999995</td>\n",
       "      <td>0.999883</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999732</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.885245</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1271705</th>\n",
       "      <td>0.751246</td>\n",
       "      <td>0.988915</td>\n",
       "      <td>0.995684</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.995473</td>\n",
       "      <td>0.967483</td>\n",
       "      <td>0.991587</td>\n",
       "      <td>0.995678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1447911</th>\n",
       "      <td>0.740257</td>\n",
       "      <td>0.988649</td>\n",
       "      <td>0.992978</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.992696</td>\n",
       "      <td>0.965080</td>\n",
       "      <td>0.989074</td>\n",
       "      <td>0.992987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1630695</th>\n",
       "      <td>0.994513</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999983</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999321</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.971515</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>692418</th>\n",
       "      <td>0.991930</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999635</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999296</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.970392</td>\n",
       "      <td>0.998754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995910</th>\n",
       "      <td>0.999995</td>\n",
       "      <td>0.999883</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999735</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.881195</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>569325</th>\n",
       "      <td>0.751246</td>\n",
       "      <td>0.988915</td>\n",
       "      <td>0.995684</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.995473</td>\n",
       "      <td>0.967483</td>\n",
       "      <td>0.991587</td>\n",
       "      <td>0.995678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1738359</th>\n",
       "      <td>0.999995</td>\n",
       "      <td>0.999883</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999449</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.842995</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1115354</th>\n",
       "      <td>0.994522</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999832</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999321</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.971520</td>\n",
       "      <td>0.999734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24529</th>\n",
       "      <td>0.999995</td>\n",
       "      <td>0.999958</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999446</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.783137</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1430006</th>\n",
       "      <td>0.999995</td>\n",
       "      <td>0.999934</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999446</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.776311</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1428167</th>\n",
       "      <td>0.994526</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999898</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999321</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.971522</td>\n",
       "      <td>0.999890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1162009</th>\n",
       "      <td>0.999995</td>\n",
       "      <td>0.999958</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999446</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.783137</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>556738</th>\n",
       "      <td>0.999995</td>\n",
       "      <td>0.999934</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999446</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.776311</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1165891</th>\n",
       "      <td>0.999996</td>\n",
       "      <td>0.999759</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.998949</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.889775</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167671</th>\n",
       "      <td>0.991822</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999749</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999296</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.970353</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152280</th>\n",
       "      <td>0.999995</td>\n",
       "      <td>0.999934</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999446</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.776311</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>765193</th>\n",
       "      <td>0.740257</td>\n",
       "      <td>0.988649</td>\n",
       "      <td>0.992978</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.992696</td>\n",
       "      <td>0.965080</td>\n",
       "      <td>0.989074</td>\n",
       "      <td>0.992987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1076231</th>\n",
       "      <td>0.990927</td>\n",
       "      <td>0.998725</td>\n",
       "      <td>0.999998</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999539</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.855889</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>314650</th>\n",
       "      <td>0.991846</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999712</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999296</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.970361</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1168461</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.973550</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.000050</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999799</td>\n",
       "      <td>0.974260</td>\n",
       "      <td>0.998615</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>580408</th>\n",
       "      <td>0.991866</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999752</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999296</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.970369</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1512941</th>\n",
       "      <td>0.995489</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999112</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.998868</td>\n",
       "      <td>0.939270</td>\n",
       "      <td>0.972069</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868517</th>\n",
       "      <td>0.991883</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999594</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999296</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.970375</td>\n",
       "      <td>0.999030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1330447</th>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.999759</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999282</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.914439</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>717391</th>\n",
       "      <td>0.989262</td>\n",
       "      <td>0.994980</td>\n",
       "      <td>0.999998</td>\n",
       "      <td>0.601045</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999789</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.892283</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1761504</th>\n",
       "      <td>0.751246</td>\n",
       "      <td>0.988915</td>\n",
       "      <td>0.995938</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.995480</td>\n",
       "      <td>0.967483</td>\n",
       "      <td>0.990953</td>\n",
       "      <td>0.995955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1485962</th>\n",
       "      <td>0.751246</td>\n",
       "      <td>0.988915</td>\n",
       "      <td>0.995684</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.995473</td>\n",
       "      <td>0.967483</td>\n",
       "      <td>0.991587</td>\n",
       "      <td>0.995678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>840229</th>\n",
       "      <td>0.994515</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999916</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999321</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.971516</td>\n",
       "      <td>0.999886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>779715</th>\n",
       "      <td>0.751246</td>\n",
       "      <td>0.988915</td>\n",
       "      <td>0.995938</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.995480</td>\n",
       "      <td>0.967483</td>\n",
       "      <td>0.990953</td>\n",
       "      <td>0.995955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>514908</th>\n",
       "      <td>0.556830</td>\n",
       "      <td>0.998437</td>\n",
       "      <td>0.999123</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999128</td>\n",
       "      <td>0.968429</td>\n",
       "      <td>0.361083</td>\n",
       "      <td>0.999123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1529642</th>\n",
       "      <td>0.740257</td>\n",
       "      <td>0.988649</td>\n",
       "      <td>0.992978</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.992696</td>\n",
       "      <td>0.965080</td>\n",
       "      <td>0.989074</td>\n",
       "      <td>0.992987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>645327</th>\n",
       "      <td>0.994520</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999905</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999321</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.971519</td>\n",
       "      <td>0.999880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1191906</th>\n",
       "      <td>0.993730</td>\n",
       "      <td>0.999013</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.601045</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999807</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.908757</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1650983</th>\n",
       "      <td>0.751246</td>\n",
       "      <td>0.988915</td>\n",
       "      <td>0.995684</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.995473</td>\n",
       "      <td>0.967483</td>\n",
       "      <td>0.991587</td>\n",
       "      <td>0.995678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114564</th>\n",
       "      <td>0.994513</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999983</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999321</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.971515</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>625802</th>\n",
       "      <td>0.995481</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999848</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999322</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.972064</td>\n",
       "      <td>0.999875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223057</th>\n",
       "      <td>0.999995</td>\n",
       "      <td>0.999934</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999446</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.776311</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>733081</th>\n",
       "      <td>0.994526</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999898</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999321</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.971522</td>\n",
       "      <td>0.999890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>789863</th>\n",
       "      <td>0.991866</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999752</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999296</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.970369</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1664682</th>\n",
       "      <td>0.751246</td>\n",
       "      <td>0.988915</td>\n",
       "      <td>0.995684</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.995473</td>\n",
       "      <td>0.967483</td>\n",
       "      <td>0.991587</td>\n",
       "      <td>0.995678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>820474</th>\n",
       "      <td>0.999995</td>\n",
       "      <td>0.999934</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999446</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.776311</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>368187</th>\n",
       "      <td>0.751246</td>\n",
       "      <td>0.988915</td>\n",
       "      <td>0.995938</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.995480</td>\n",
       "      <td>0.967483</td>\n",
       "      <td>0.990953</td>\n",
       "      <td>0.995955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>653553</th>\n",
       "      <td>0.995475</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999764</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999322</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.972061</td>\n",
       "      <td>0.999864</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140369</th>\n",
       "      <td>0.994515</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999945</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999321</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.971517</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1733900</th>\n",
       "      <td>0.740257</td>\n",
       "      <td>0.988649</td>\n",
       "      <td>0.992978</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.992696</td>\n",
       "      <td>0.965080</td>\n",
       "      <td>0.989074</td>\n",
       "      <td>0.992987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1655707</th>\n",
       "      <td>0.999995</td>\n",
       "      <td>0.999934</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999446</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.776311</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>976048</th>\n",
       "      <td>0.991902</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999743</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999296</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.970382</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1395958</th>\n",
       "      <td>0.999964</td>\n",
       "      <td>0.900396</td>\n",
       "      <td>0.999985</td>\n",
       "      <td>0.562223</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.994368</td>\n",
       "      <td>0.974542</td>\n",
       "      <td>0.958311</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              dnn        rf      lgbm       ada       svm       cat       xgb  \\\n",
       "567509   0.994514  0.995236  0.999923  0.430217  0.333333  0.999321  0.971597   \n",
       "34764    0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
       "991802   0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
       "647470   0.991945  0.995236  0.999790  0.430217  0.333333  0.999296  0.971597   \n",
       "1260575  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
       "1141227  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
       "1774849  0.999995  0.999883  0.999999  0.559890  0.333333  0.999732  0.974612   \n",
       "1271705  0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
       "1447911  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
       "1630695  0.994513  0.995236  0.999983  0.430217  0.333333  0.999321  0.971597   \n",
       "692418   0.991930  0.995236  0.999635  0.430217  0.333333  0.999296  0.971597   \n",
       "995910   0.999995  0.999883  0.999999  0.559890  0.333333  0.999735  0.974612   \n",
       "569325   0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
       "1738359  0.999995  0.999883  0.999999  0.559890  0.333333  0.999449  0.974612   \n",
       "1115354  0.994522  0.995236  0.999832  0.430217  0.333333  0.999321  0.971597   \n",
       "24529    0.999995  0.999958  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
       "1430006  0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
       "1428167  0.994526  0.995236  0.999898  0.430217  0.333333  0.999321  0.971597   \n",
       "1162009  0.999995  0.999958  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
       "556738   0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
       "1165891  0.999996  0.999759  0.999999  0.559890  0.333333  0.998949  0.974612   \n",
       "167671   0.991822  0.995236  0.999749  0.430217  0.333333  0.999296  0.971597   \n",
       "152280   0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
       "765193   0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
       "1076231  0.990927  0.998725  0.999998  0.559890  0.333333  0.999539  0.974612   \n",
       "314650   0.991846  0.995236  0.999712  0.430217  0.333333  0.999296  0.971597   \n",
       "1168461  1.000000  0.973550  0.999999  0.000050  0.333333  0.999799  0.974260   \n",
       "580408   0.991866  0.995236  0.999752  0.430217  0.333333  0.999296  0.971597   \n",
       "1512941  0.995489  0.995236  0.999112  0.430217  0.333333  0.998868  0.939270   \n",
       "868517   0.991883  0.995236  0.999594  0.430217  0.333333  0.999296  0.971597   \n",
       "1330447  0.999999  0.999759  0.999999  0.559890  0.333333  0.999282  0.974612   \n",
       "717391   0.989262  0.994980  0.999998  0.601045  0.333333  0.999789  0.974612   \n",
       "1761504  0.751246  0.988915  0.995938  0.530801  0.333333  0.995480  0.967483   \n",
       "1485962  0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
       "840229   0.994515  0.995236  0.999916  0.430217  0.333333  0.999321  0.971597   \n",
       "779715   0.751246  0.988915  0.995938  0.530801  0.333333  0.995480  0.967483   \n",
       "514908   0.556830  0.998437  0.999123  0.559890  0.333333  0.999128  0.968429   \n",
       "1529642  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
       "645327   0.994520  0.995236  0.999905  0.430217  0.333333  0.999321  0.971597   \n",
       "1191906  0.993730  0.999013  0.999999  0.601045  0.333333  0.999807  0.974612   \n",
       "1650983  0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
       "114564   0.994513  0.995236  0.999983  0.430217  0.333333  0.999321  0.971597   \n",
       "625802   0.995481  0.995236  0.999848  0.430217  0.333333  0.999322  0.971597   \n",
       "223057   0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
       "733081   0.994526  0.995236  0.999898  0.430217  0.333333  0.999321  0.971597   \n",
       "789863   0.991866  0.995236  0.999752  0.430217  0.333333  0.999296  0.971597   \n",
       "1664682  0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
       "820474   0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
       "368187   0.751246  0.988915  0.995938  0.530801  0.333333  0.995480  0.967483   \n",
       "653553   0.995475  0.995236  0.999764  0.430217  0.333333  0.999322  0.971597   \n",
       "140369   0.994515  0.995236  0.999945  0.430217  0.333333  0.999321  0.971597   \n",
       "1733900  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
       "1655707  0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
       "976048   0.991902  0.995236  0.999743  0.430217  0.333333  0.999296  0.971597   \n",
       "1395958  0.999964  0.900396  0.999985  0.562223  0.333333  0.994368  0.974542   \n",
       "\n",
       "               lr        dt  \n",
       "567509   0.971516  0.999925  \n",
       "34764    0.989074  0.992987  \n",
       "991802   0.991587  0.995678  \n",
       "647470   0.970398  1.000000  \n",
       "1260575  0.989074  0.992987  \n",
       "1141227  0.989074  0.992987  \n",
       "1774849  0.885245  1.000000  \n",
       "1271705  0.991587  0.995678  \n",
       "1447911  0.989074  0.992987  \n",
       "1630695  0.971515  1.000000  \n",
       "692418   0.970392  0.998754  \n",
       "995910   0.881195  1.000000  \n",
       "569325   0.991587  0.995678  \n",
       "1738359  0.842995  1.000000  \n",
       "1115354  0.971520  0.999734  \n",
       "24529    0.783137  1.000000  \n",
       "1430006  0.776311  1.000000  \n",
       "1428167  0.971522  0.999890  \n",
       "1162009  0.783137  1.000000  \n",
       "556738   0.776311  1.000000  \n",
       "1165891  0.889775  1.000000  \n",
       "167671   0.970353  1.000000  \n",
       "152280   0.776311  1.000000  \n",
       "765193   0.989074  0.992987  \n",
       "1076231  0.855889  1.000000  \n",
       "314650   0.970361  1.000000  \n",
       "1168461  0.998615  1.000000  \n",
       "580408   0.970369  1.000000  \n",
       "1512941  0.972069  1.000000  \n",
       "868517   0.970375  0.999030  \n",
       "1330447  0.914439  1.000000  \n",
       "717391   0.892283  1.000000  \n",
       "1761504  0.990953  0.995955  \n",
       "1485962  0.991587  0.995678  \n",
       "840229   0.971516  0.999886  \n",
       "779715   0.990953  0.995955  \n",
       "514908   0.361083  0.999123  \n",
       "1529642  0.989074  0.992987  \n",
       "645327   0.971519  0.999880  \n",
       "1191906  0.908757  1.000000  \n",
       "1650983  0.991587  0.995678  \n",
       "114564   0.971515  1.000000  \n",
       "625802   0.972064  0.999875  \n",
       "223057   0.776311  1.000000  \n",
       "733081   0.971522  0.999890  \n",
       "789863   0.970369  1.000000  \n",
       "1664682  0.991587  0.995678  \n",
       "820474   0.776311  1.000000  \n",
       "368187   0.990953  0.995955  \n",
       "653553   0.972061  0.999864  \n",
       "140369   0.971517  1.000000  \n",
       "1733900  0.989074  0.992987  \n",
       "1655707  0.776311  1.000000  \n",
       "976048   0.970382  1.000000  \n",
       "1395958  0.958311  1.000000  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_01"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0   1.0   2.0\n",
      "0.0  19.0   1.0   0.0\n",
      "1.0   1.0  18.0   0.0\n",
      "2.0   0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9636363636363636\n",
      "Precision total:  0.9657894736842105\n",
      "Recall total:  0.9657894736842105\n",
      "F1 total:  0.9657894736842105\n",
      "BACC total:  0.9657894736842105\n",
      "MCC total:  0.9452191235059761\n",
      "0.031017303466796875\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "start = time.time()\n",
    "\n",
    "# Create a Decision Tree Classifier\n",
    "dt_classifier = DecisionTreeClassifier(random_state=42)\n",
    "# Train the classifier on the training data\n",
    "dt_classifier.fit(X_train_01, y_train_01)\n",
    "# Make predictions on the test data\n",
    "preds_dt = dt_classifier.predict(X_test_01)\n",
    "# Evaluate the accuracy of the model\n",
    "preds_dt_prob = dt_classifier.predict_proba(X_test_01)\n",
    "\n",
    "\n",
    "pred_label = preds_dt\n",
    "name = 'dt'\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n",
    "print(time_taken)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Voting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "    \n",
    "if pick_prob == 0:\n",
    "    # Voting start\n",
    "\n",
    "    import pandas as pd\n",
    "    from scipy.stats import mode\n",
    "\n",
    "    # Assuming 'df' is your original DataFrame with columns 'dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm', 'label'\n",
    "    df = X_test_01\n",
    "    # Extract predictions columns\n",
    "    \n",
    "    # predictions = df[['dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm','cat','xgb']]\n",
    "        # selected_columns = df.loc[:, ~df.columns.isin(['rf'])]\n",
    "    predictions = df.loc[:, ~df.columns.isin(['label'])] #df[column_features]\n",
    "\n",
    "    # Use the mode function along axis 1 to get the most common prediction for each row\n",
    "    ensemble_predictions, _ = mode(predictions.values, axis=1)\n",
    "\n",
    "    # Add the ensemble predictions to the DataFrame\n",
    "    df['ensemble'] = ensemble_predictions.astype(int)\n",
    "\n",
    "    # Display the DataFrame with ensemble predictions\n",
    "    print(df)\n",
    "\n",
    "    pred_label = df ['ensemble'].values\n",
    "    df.pop('ensemble')\n",
    "\n",
    "    #testing metrics def\n",
    "    name = 'voting'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "   \n",
    "else:\n",
    "    name = 'voting'\n",
    "    globals()[f\"{name}_acc_01\"] = 0\n",
    "    globals()[f\"{name}_pre_01\"] = 0\n",
    "    globals()[f\"{name}_rec_01\"] = 0\n",
    "    globals()[f\"{name}_f1_01\"] = 0\n",
    "    globals()[f\"{name}_bacc_01\"] = 0\n",
    "    globals()[f\"{name}_mcc_01\"] = 0\n",
    "    globals()[f\"{name}_time_01\"] = 9999\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voting_acc_01\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              dnn        rf      lgbm       ada       svm       cat       xgb  \\\n",
      "567509   0.994514  0.995236  0.999923  0.430217  0.333333  0.999321  0.971597   \n",
      "34764    0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "991802   0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
      "647470   0.991945  0.995236  0.999790  0.430217  0.333333  0.999296  0.971597   \n",
      "1260575  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "1141227  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "1774849  0.999995  0.999883  0.999999  0.559890  0.333333  0.999732  0.974612   \n",
      "1271705  0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
      "1447911  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "1630695  0.994513  0.995236  0.999983  0.430217  0.333333  0.999321  0.971597   \n",
      "692418   0.991930  0.995236  0.999635  0.430217  0.333333  0.999296  0.971597   \n",
      "995910   0.999995  0.999883  0.999999  0.559890  0.333333  0.999735  0.974612   \n",
      "569325   0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
      "1738359  0.999995  0.999883  0.999999  0.559890  0.333333  0.999449  0.974612   \n",
      "1115354  0.994522  0.995236  0.999832  0.430217  0.333333  0.999321  0.971597   \n",
      "24529    0.999995  0.999958  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "1430006  0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "1428167  0.994526  0.995236  0.999898  0.430217  0.333333  0.999321  0.971597   \n",
      "1162009  0.999995  0.999958  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "556738   0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "1165891  0.999996  0.999759  0.999999  0.559890  0.333333  0.998949  0.974612   \n",
      "167671   0.991822  0.995236  0.999749  0.430217  0.333333  0.999296  0.971597   \n",
      "152280   0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "765193   0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "1076231  0.990927  0.998725  0.999998  0.559890  0.333333  0.999539  0.974612   \n",
      "314650   0.991846  0.995236  0.999712  0.430217  0.333333  0.999296  0.971597   \n",
      "1168461  1.000000  0.973550  0.999999  0.000050  0.333333  0.999799  0.974260   \n",
      "580408   0.991866  0.995236  0.999752  0.430217  0.333333  0.999296  0.971597   \n",
      "1512941  0.995489  0.995236  0.999112  0.430217  0.333333  0.998868  0.939270   \n",
      "868517   0.991883  0.995236  0.999594  0.430217  0.333333  0.999296  0.971597   \n",
      "1330447  0.999999  0.999759  0.999999  0.559890  0.333333  0.999282  0.974612   \n",
      "717391   0.989262  0.994980  0.999998  0.601045  0.333333  0.999789  0.974612   \n",
      "1761504  0.751246  0.988915  0.995938  0.530801  0.333333  0.995480  0.967483   \n",
      "1485962  0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
      "840229   0.994515  0.995236  0.999916  0.430217  0.333333  0.999321  0.971597   \n",
      "779715   0.751246  0.988915  0.995938  0.530801  0.333333  0.995480  0.967483   \n",
      "514908   0.556830  0.998437  0.999123  0.559890  0.333333  0.999128  0.968429   \n",
      "1529642  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "645327   0.994520  0.995236  0.999905  0.430217  0.333333  0.999321  0.971597   \n",
      "1191906  0.993730  0.999013  0.999999  0.601045  0.333333  0.999807  0.974612   \n",
      "1650983  0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
      "114564   0.994513  0.995236  0.999983  0.430217  0.333333  0.999321  0.971597   \n",
      "625802   0.995481  0.995236  0.999848  0.430217  0.333333  0.999322  0.971597   \n",
      "223057   0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "733081   0.994526  0.995236  0.999898  0.430217  0.333333  0.999321  0.971597   \n",
      "789863   0.991866  0.995236  0.999752  0.430217  0.333333  0.999296  0.971597   \n",
      "1664682  0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
      "820474   0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "368187   0.751246  0.988915  0.995938  0.530801  0.333333  0.995480  0.967483   \n",
      "653553   0.995475  0.995236  0.999764  0.430217  0.333333  0.999322  0.971597   \n",
      "140369   0.994515  0.995236  0.999945  0.430217  0.333333  0.999321  0.971597   \n",
      "1733900  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "1655707  0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "976048   0.991902  0.995236  0.999743  0.430217  0.333333  0.999296  0.971597   \n",
      "1395958  0.999964  0.900396  0.999985  0.562223  0.333333  0.994368  0.974542   \n",
      "\n",
      "               lr        dt  results  \n",
      "567509   0.971516  0.999925        1  \n",
      "34764    0.989074  0.992987        1  \n",
      "991802   0.991587  0.995678        1  \n",
      "647470   0.970398  1.000000        1  \n",
      "1260575  0.989074  0.992987        1  \n",
      "1141227  0.989074  0.992987        1  \n",
      "1774849  0.885245  1.000000        1  \n",
      "1271705  0.991587  0.995678        1  \n",
      "1447911  0.989074  0.992987        1  \n",
      "1630695  0.971515  1.000000        1  \n",
      "692418   0.970392  0.998754        1  \n",
      "995910   0.881195  1.000000        1  \n",
      "569325   0.991587  0.995678        1  \n",
      "1738359  0.842995  1.000000        1  \n",
      "1115354  0.971520  0.999734        1  \n",
      "24529    0.783137  1.000000        1  \n",
      "1430006  0.776311  1.000000        1  \n",
      "1428167  0.971522  0.999890        1  \n",
      "1162009  0.783137  1.000000        1  \n",
      "556738   0.776311  1.000000        1  \n",
      "1165891  0.889775  1.000000        1  \n",
      "167671   0.970353  1.000000        1  \n",
      "152280   0.776311  1.000000        1  \n",
      "765193   0.989074  0.992987        1  \n",
      "1076231  0.855889  1.000000        1  \n",
      "314650   0.970361  1.000000        1  \n",
      "1168461  0.998615  1.000000        1  \n",
      "580408   0.970369  1.000000        1  \n",
      "1512941  0.972069  1.000000        1  \n",
      "868517   0.970375  0.999030        1  \n",
      "1330447  0.914439  1.000000        1  \n",
      "717391   0.892283  1.000000        1  \n",
      "1761504  0.990953  0.995955        1  \n",
      "1485962  0.991587  0.995678        1  \n",
      "840229   0.971516  0.999886        1  \n",
      "779715   0.990953  0.995955        1  \n",
      "514908   0.361083  0.999123        1  \n",
      "1529642  0.989074  0.992987        1  \n",
      "645327   0.971519  0.999880        1  \n",
      "1191906  0.908757  1.000000        1  \n",
      "1650983  0.991587  0.995678        1  \n",
      "114564   0.971515  1.000000        1  \n",
      "625802   0.972064  0.999875        1  \n",
      "223057   0.776311  1.000000        1  \n",
      "733081   0.971522  0.999890        1  \n",
      "789863   0.970369  1.000000        1  \n",
      "1664682  0.991587  0.995678        1  \n",
      "820474   0.776311  1.000000        1  \n",
      "368187   0.990953  0.995955        1  \n",
      "653553   0.972061  0.999864        1  \n",
      "140369   0.971517  1.000000        1  \n",
      "1733900  0.989074  0.992987        1  \n",
      "1655707  0.776311  1.000000        1  \n",
      "976048   0.970382  1.000000        1  \n",
      "1395958  0.958311  1.000000        1  \n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0  1.0  2.0\n",
      "0.0  20.0  0.0  0.0\n",
      "1.0  19.0  0.0  0.0\n",
      "2.0  16.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.34545454545454546\n",
      "Precision total:  0.11515151515151516\n",
      "Recall total:  0.3333333333333333\n",
      "F1 total:  0.17117117117117117\n",
      "BACC total:  0.3333333333333333\n",
      "MCC total:  0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "invalid value encountered in double_scalars\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# if pick_prob == 0:\n",
    "if 0 == 0:\n",
    "    # Average start\n",
    "\n",
    "    import pandas as pd\n",
    "    from scipy.stats import mode\n",
    "\n",
    "    # Assuming 'df' is your original DataFrame with columns 'dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm', 'label'\n",
    "    df = X_test_01\n",
    "    predictions = df.loc[:, ~df.columns.isin(['label'])] #df[column_features]\n",
    "\n",
    "   \n",
    "\n",
    "    column_sums = df.sum(axis=1)\n",
    "    row_average = df.mean(axis=1)\n",
    "\n",
    "    # Approximate the result to the closest integer\n",
    "    rounded_average = row_average.round().astype(int)\n",
    "\n",
    "    # print(rounded_average)\n",
    "\n",
    "    df['results'] = rounded_average\n",
    "    print(df)\n",
    " \n",
    "    pred_label = df ['results'].values\n",
    "\n",
    "    # pred_label = df ['ensemble'].values\n",
    "    # df.pop('ensemble')\n",
    "    df.pop('results')\n",
    "\n",
    "    # df.pop('column_sums')\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    name = 'avg'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    \n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Weighed Average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# column_features\n",
    "# move this up with column_features\n",
    "\n",
    "#important update this as you need to select the important features,\n",
    "#  the left is the least important while the right is the most important\n",
    "# needs automation\n",
    "if pick_prob == 1 and feature_selection_bit == 1:\n",
    "    column_features = column_features1\n",
    "\n",
    "elif pick_prob == 0 and feature_selection_bit == 1: \n",
    "    column_features = column_features0\n",
    "\n",
    "else: None\n",
    "\n",
    "feature_selection_columns_in_order_of_importance = column_features[:-1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.0, 0.125, 0.25, 0.375, 0.5, 0.625, 0.75, 0.875, 1.0]\n",
      "[0.0, 0.125, 0.25, 0.375, 0.5, 0.625, 0.75, 0.875, 1.0]\n",
      "              dnn        rf      lgbm       ada       svm       cat       xgb  \\\n",
      "567509   0.994514  0.995236  0.999923  0.430217  0.333333  0.999321  0.971597   \n",
      "34764    0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "991802   0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
      "647470   0.991945  0.995236  0.999790  0.430217  0.333333  0.999296  0.971597   \n",
      "1260575  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "1141227  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "1774849  0.999995  0.999883  0.999999  0.559890  0.333333  0.999732  0.974612   \n",
      "1271705  0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
      "1447911  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "1630695  0.994513  0.995236  0.999983  0.430217  0.333333  0.999321  0.971597   \n",
      "692418   0.991930  0.995236  0.999635  0.430217  0.333333  0.999296  0.971597   \n",
      "995910   0.999995  0.999883  0.999999  0.559890  0.333333  0.999735  0.974612   \n",
      "569325   0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
      "1738359  0.999995  0.999883  0.999999  0.559890  0.333333  0.999449  0.974612   \n",
      "1115354  0.994522  0.995236  0.999832  0.430217  0.333333  0.999321  0.971597   \n",
      "24529    0.999995  0.999958  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "1430006  0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "1428167  0.994526  0.995236  0.999898  0.430217  0.333333  0.999321  0.971597   \n",
      "1162009  0.999995  0.999958  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "556738   0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "1165891  0.999996  0.999759  0.999999  0.559890  0.333333  0.998949  0.974612   \n",
      "167671   0.991822  0.995236  0.999749  0.430217  0.333333  0.999296  0.971597   \n",
      "152280   0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "765193   0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "1076231  0.990927  0.998725  0.999998  0.559890  0.333333  0.999539  0.974612   \n",
      "314650   0.991846  0.995236  0.999712  0.430217  0.333333  0.999296  0.971597   \n",
      "1168461  1.000000  0.973550  0.999999  0.000050  0.333333  0.999799  0.974260   \n",
      "580408   0.991866  0.995236  0.999752  0.430217  0.333333  0.999296  0.971597   \n",
      "1512941  0.995489  0.995236  0.999112  0.430217  0.333333  0.998868  0.939270   \n",
      "868517   0.991883  0.995236  0.999594  0.430217  0.333333  0.999296  0.971597   \n",
      "1330447  0.999999  0.999759  0.999999  0.559890  0.333333  0.999282  0.974612   \n",
      "717391   0.989262  0.994980  0.999998  0.601045  0.333333  0.999789  0.974612   \n",
      "1761504  0.751246  0.988915  0.995938  0.530801  0.333333  0.995480  0.967483   \n",
      "1485962  0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
      "840229   0.994515  0.995236  0.999916  0.430217  0.333333  0.999321  0.971597   \n",
      "779715   0.751246  0.988915  0.995938  0.530801  0.333333  0.995480  0.967483   \n",
      "514908   0.556830  0.998437  0.999123  0.559890  0.333333  0.999128  0.968429   \n",
      "1529642  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "645327   0.994520  0.995236  0.999905  0.430217  0.333333  0.999321  0.971597   \n",
      "1191906  0.993730  0.999013  0.999999  0.601045  0.333333  0.999807  0.974612   \n",
      "1650983  0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
      "114564   0.994513  0.995236  0.999983  0.430217  0.333333  0.999321  0.971597   \n",
      "625802   0.995481  0.995236  0.999848  0.430217  0.333333  0.999322  0.971597   \n",
      "223057   0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "733081   0.994526  0.995236  0.999898  0.430217  0.333333  0.999321  0.971597   \n",
      "789863   0.991866  0.995236  0.999752  0.430217  0.333333  0.999296  0.971597   \n",
      "1664682  0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
      "820474   0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "368187   0.751246  0.988915  0.995938  0.530801  0.333333  0.995480  0.967483   \n",
      "653553   0.995475  0.995236  0.999764  0.430217  0.333333  0.999322  0.971597   \n",
      "140369   0.994515  0.995236  0.999945  0.430217  0.333333  0.999321  0.971597   \n",
      "1733900  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "1655707  0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "976048   0.991902  0.995236  0.999743  0.430217  0.333333  0.999296  0.971597   \n",
      "1395958  0.999964  0.900396  0.999985  0.562223  0.333333  0.994368  0.974542   \n",
      "\n",
      "               lr        dt  \n",
      "567509   0.971516  0.999925  \n",
      "34764    0.989074  0.992987  \n",
      "991802   0.991587  0.995678  \n",
      "647470   0.970398  1.000000  \n",
      "1260575  0.989074  0.992987  \n",
      "1141227  0.989074  0.992987  \n",
      "1774849  0.885245  1.000000  \n",
      "1271705  0.991587  0.995678  \n",
      "1447911  0.989074  0.992987  \n",
      "1630695  0.971515  1.000000  \n",
      "692418   0.970392  0.998754  \n",
      "995910   0.881195  1.000000  \n",
      "569325   0.991587  0.995678  \n",
      "1738359  0.842995  1.000000  \n",
      "1115354  0.971520  0.999734  \n",
      "24529    0.783137  1.000000  \n",
      "1430006  0.776311  1.000000  \n",
      "1428167  0.971522  0.999890  \n",
      "1162009  0.783137  1.000000  \n",
      "556738   0.776311  1.000000  \n",
      "1165891  0.889775  1.000000  \n",
      "167671   0.970353  1.000000  \n",
      "152280   0.776311  1.000000  \n",
      "765193   0.989074  0.992987  \n",
      "1076231  0.855889  1.000000  \n",
      "314650   0.970361  1.000000  \n",
      "1168461  0.998615  1.000000  \n",
      "580408   0.970369  1.000000  \n",
      "1512941  0.972069  1.000000  \n",
      "868517   0.970375  0.999030  \n",
      "1330447  0.914439  1.000000  \n",
      "717391   0.892283  1.000000  \n",
      "1761504  0.990953  0.995955  \n",
      "1485962  0.991587  0.995678  \n",
      "840229   0.971516  0.999886  \n",
      "779715   0.990953  0.995955  \n",
      "514908   0.361083  0.999123  \n",
      "1529642  0.989074  0.992987  \n",
      "645327   0.971519  0.999880  \n",
      "1191906  0.908757  1.000000  \n",
      "1650983  0.991587  0.995678  \n",
      "114564   0.971515  1.000000  \n",
      "625802   0.972064  0.999875  \n",
      "223057   0.776311  1.000000  \n",
      "733081   0.971522  0.999890  \n",
      "789863   0.970369  1.000000  \n",
      "1664682  0.991587  0.995678  \n",
      "820474   0.776311  1.000000  \n",
      "368187   0.990953  0.995955  \n",
      "653553   0.972061  0.999864  \n",
      "140369   0.971517  1.000000  \n",
      "1733900  0.989074  0.992987  \n",
      "1655707  0.776311  1.000000  \n",
      "976048   0.970382  1.000000  \n",
      "1395958  0.958311  1.000000  \n",
      "567509     0.867924\n",
      "34764      0.875603\n",
      "991802     0.877634\n",
      "647470     0.867712\n",
      "1260575    0.875603\n",
      "1141227    0.875603\n",
      "1774849    0.862665\n",
      "1271705    0.877634\n",
      "1447911    0.875603\n",
      "1630695    0.867944\n",
      "692418     0.867426\n",
      "995910     0.861878\n",
      "569325     0.877634\n",
      "1738359    0.854410\n",
      "1115354    0.867877\n",
      "24529      0.842773\n",
      "1430006    0.841445\n",
      "1428167    0.867916\n",
      "1162009    0.842773\n",
      "556738     0.841445\n",
      "1165891    0.863433\n",
      "167671     0.867701\n",
      "152280     0.841445\n",
      "765193     0.875603\n",
      "1076231    0.856898\n",
      "314650     0.867701\n",
      "1168461    0.837275\n",
      "580408     0.867705\n",
      "1512941    0.862553\n",
      "868517     0.867481\n",
      "1330447    0.868275\n",
      "717391     0.867335\n",
      "1761504    0.877587\n",
      "1485962    0.877634\n",
      "840229     0.867915\n",
      "779715     0.877587\n",
      "514908     0.759346\n",
      "1529642    0.875603\n",
      "645327     0.867914\n",
      "1191906    0.870652\n",
      "1650983    0.877634\n",
      "114564     0.867944\n",
      "625802     0.868015\n",
      "223057     0.841445\n",
      "733081     0.867916\n",
      "789863     0.867705\n",
      "1664682    0.877634\n",
      "820474     0.841445\n",
      "368187     0.877587\n",
      "653553     0.868008\n",
      "140369     0.867942\n",
      "1733900    0.875603\n",
      "1655707    0.841445\n",
      "976048     0.867707\n",
      "1395958    0.873545\n",
      "dtype: float64\n",
      "567509     1\n",
      "34764      1\n",
      "991802     1\n",
      "647470     1\n",
      "1260575    1\n",
      "1141227    1\n",
      "1774849    1\n",
      "1271705    1\n",
      "1447911    1\n",
      "1630695    1\n",
      "692418     1\n",
      "995910     1\n",
      "569325     1\n",
      "1738359    1\n",
      "1115354    1\n",
      "24529      1\n",
      "1430006    1\n",
      "1428167    1\n",
      "1162009    1\n",
      "556738     1\n",
      "1165891    1\n",
      "167671     1\n",
      "152280     1\n",
      "765193     1\n",
      "1076231    1\n",
      "314650     1\n",
      "1168461    1\n",
      "580408     1\n",
      "1512941    1\n",
      "868517     1\n",
      "1330447    1\n",
      "717391     1\n",
      "1761504    1\n",
      "1485962    1\n",
      "840229     1\n",
      "779715     1\n",
      "514908     1\n",
      "1529642    1\n",
      "645327     1\n",
      "1191906    1\n",
      "1650983    1\n",
      "114564     1\n",
      "625802     1\n",
      "223057     1\n",
      "733081     1\n",
      "789863     1\n",
      "1664682    1\n",
      "820474     1\n",
      "368187     1\n",
      "653553     1\n",
      "140369     1\n",
      "1733900    1\n",
      "1655707    1\n",
      "976048     1\n",
      "1395958    1\n",
      "dtype: int64\n",
      "              dnn        rf      lgbm       ada       svm       cat       xgb  \\\n",
      "567509   0.994514  0.995236  0.999923  0.430217  0.333333  0.999321  0.971597   \n",
      "34764    0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "991802   0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
      "647470   0.991945  0.995236  0.999790  0.430217  0.333333  0.999296  0.971597   \n",
      "1260575  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "1141227  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "1774849  0.999995  0.999883  0.999999  0.559890  0.333333  0.999732  0.974612   \n",
      "1271705  0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
      "1447911  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "1630695  0.994513  0.995236  0.999983  0.430217  0.333333  0.999321  0.971597   \n",
      "692418   0.991930  0.995236  0.999635  0.430217  0.333333  0.999296  0.971597   \n",
      "995910   0.999995  0.999883  0.999999  0.559890  0.333333  0.999735  0.974612   \n",
      "569325   0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
      "1738359  0.999995  0.999883  0.999999  0.559890  0.333333  0.999449  0.974612   \n",
      "1115354  0.994522  0.995236  0.999832  0.430217  0.333333  0.999321  0.971597   \n",
      "24529    0.999995  0.999958  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "1430006  0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "1428167  0.994526  0.995236  0.999898  0.430217  0.333333  0.999321  0.971597   \n",
      "1162009  0.999995  0.999958  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "556738   0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "1165891  0.999996  0.999759  0.999999  0.559890  0.333333  0.998949  0.974612   \n",
      "167671   0.991822  0.995236  0.999749  0.430217  0.333333  0.999296  0.971597   \n",
      "152280   0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "765193   0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "1076231  0.990927  0.998725  0.999998  0.559890  0.333333  0.999539  0.974612   \n",
      "314650   0.991846  0.995236  0.999712  0.430217  0.333333  0.999296  0.971597   \n",
      "1168461  1.000000  0.973550  0.999999  0.000050  0.333333  0.999799  0.974260   \n",
      "580408   0.991866  0.995236  0.999752  0.430217  0.333333  0.999296  0.971597   \n",
      "1512941  0.995489  0.995236  0.999112  0.430217  0.333333  0.998868  0.939270   \n",
      "868517   0.991883  0.995236  0.999594  0.430217  0.333333  0.999296  0.971597   \n",
      "1330447  0.999999  0.999759  0.999999  0.559890  0.333333  0.999282  0.974612   \n",
      "717391   0.989262  0.994980  0.999998  0.601045  0.333333  0.999789  0.974612   \n",
      "1761504  0.751246  0.988915  0.995938  0.530801  0.333333  0.995480  0.967483   \n",
      "1485962  0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
      "840229   0.994515  0.995236  0.999916  0.430217  0.333333  0.999321  0.971597   \n",
      "779715   0.751246  0.988915  0.995938  0.530801  0.333333  0.995480  0.967483   \n",
      "514908   0.556830  0.998437  0.999123  0.559890  0.333333  0.999128  0.968429   \n",
      "1529642  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "645327   0.994520  0.995236  0.999905  0.430217  0.333333  0.999321  0.971597   \n",
      "1191906  0.993730  0.999013  0.999999  0.601045  0.333333  0.999807  0.974612   \n",
      "1650983  0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
      "114564   0.994513  0.995236  0.999983  0.430217  0.333333  0.999321  0.971597   \n",
      "625802   0.995481  0.995236  0.999848  0.430217  0.333333  0.999322  0.971597   \n",
      "223057   0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "733081   0.994526  0.995236  0.999898  0.430217  0.333333  0.999321  0.971597   \n",
      "789863   0.991866  0.995236  0.999752  0.430217  0.333333  0.999296  0.971597   \n",
      "1664682  0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
      "820474   0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "368187   0.751246  0.988915  0.995938  0.530801  0.333333  0.995480  0.967483   \n",
      "653553   0.995475  0.995236  0.999764  0.430217  0.333333  0.999322  0.971597   \n",
      "140369   0.994515  0.995236  0.999945  0.430217  0.333333  0.999321  0.971597   \n",
      "1733900  0.740257  0.988649  0.992978  0.530801  0.333333  0.992696  0.965080   \n",
      "1655707  0.999995  0.999934  0.999999  0.559890  0.333333  0.999446  0.974612   \n",
      "976048   0.991902  0.995236  0.999743  0.430217  0.333333  0.999296  0.971597   \n",
      "1395958  0.999964  0.900396  0.999985  0.562223  0.333333  0.994368  0.974542   \n",
      "\n",
      "               lr        dt  results  \n",
      "567509   0.971516  0.999925        1  \n",
      "34764    0.989074  0.992987        1  \n",
      "991802   0.991587  0.995678        1  \n",
      "647470   0.970398  1.000000        1  \n",
      "1260575  0.989074  0.992987        1  \n",
      "1141227  0.989074  0.992987        1  \n",
      "1774849  0.885245  1.000000        1  \n",
      "1271705  0.991587  0.995678        1  \n",
      "1447911  0.989074  0.992987        1  \n",
      "1630695  0.971515  1.000000        1  \n",
      "692418   0.970392  0.998754        1  \n",
      "995910   0.881195  1.000000        1  \n",
      "569325   0.991587  0.995678        1  \n",
      "1738359  0.842995  1.000000        1  \n",
      "1115354  0.971520  0.999734        1  \n",
      "24529    0.783137  1.000000        1  \n",
      "1430006  0.776311  1.000000        1  \n",
      "1428167  0.971522  0.999890        1  \n",
      "1162009  0.783137  1.000000        1  \n",
      "556738   0.776311  1.000000        1  \n",
      "1165891  0.889775  1.000000        1  \n",
      "167671   0.970353  1.000000        1  \n",
      "152280   0.776311  1.000000        1  \n",
      "765193   0.989074  0.992987        1  \n",
      "1076231  0.855889  1.000000        1  \n",
      "314650   0.970361  1.000000        1  \n",
      "1168461  0.998615  1.000000        1  \n",
      "580408   0.970369  1.000000        1  \n",
      "1512941  0.972069  1.000000        1  \n",
      "868517   0.970375  0.999030        1  \n",
      "1330447  0.914439  1.000000        1  \n",
      "717391   0.892283  1.000000        1  \n",
      "1761504  0.990953  0.995955        1  \n",
      "1485962  0.991587  0.995678        1  \n",
      "840229   0.971516  0.999886        1  \n",
      "779715   0.990953  0.995955        1  \n",
      "514908   0.361083  0.999123        1  \n",
      "1529642  0.989074  0.992987        1  \n",
      "645327   0.971519  0.999880        1  \n",
      "1191906  0.908757  1.000000        1  \n",
      "1650983  0.991587  0.995678        1  \n",
      "114564   0.971515  1.000000        1  \n",
      "625802   0.972064  0.999875        1  \n",
      "223057   0.776311  1.000000        1  \n",
      "733081   0.971522  0.999890        1  \n",
      "789863   0.970369  1.000000        1  \n",
      "1664682  0.991587  0.995678        1  \n",
      "820474   0.776311  1.000000        1  \n",
      "368187   0.990953  0.995955        1  \n",
      "653553   0.972061  0.999864        1  \n",
      "140369   0.971517  1.000000        1  \n",
      "1733900  0.989074  0.992987        1  \n",
      "1655707  0.776311  1.000000        1  \n",
      "976048   0.970382  1.000000        1  \n",
      "1395958  0.958311  1.000000        1  \n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0  1.0  2.0\n",
      "0.0  20.0  0.0  0.0\n",
      "1.0  19.0  0.0  0.0\n",
      "2.0  16.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.34545454545454546\n",
      "Precision total:  0.11515151515151516\n",
      "Recall total:  0.3333333333333333\n",
      "F1 total:  0.17117117117117117\n",
      "BACC total:  0.3333333333333333\n",
      "MCC total:  0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "invalid value encountered in double_scalars\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# if pick_prob == 0:\n",
    "if 0 == 0:\n",
    "    # Average start\n",
    "\n",
    "    import pandas as pd\n",
    "    from scipy.stats import mode\n",
    "\n",
    "    # Assuming 'df' is your original DataFrame with columns 'dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm', 'label'\n",
    "    # df = X_test_01\n",
    "    df = X_test_01[feature_selection_columns_in_order_of_importance]\n",
    "    # Extract predictions columns\n",
    "    \n",
    "    # predictions = df[['dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm','cat','xgb']]\n",
    "        # selected_columns = df.loc[:, ~df.columns.isin(['rf'])]\n",
    "    predictions = df.loc[:, ~df.columns.isin(['label'])] #df[column_features]\n",
    "\n",
    "    # weight\n",
    "    weights_values = []\n",
    "\n",
    "    # linear weight distribution\n",
    "    for i in range(0,len(~df.columns.isin(['label']))):\n",
    "        weights_values.append(i/(len(~df.columns.isin(['label']))-1))\n",
    "    print(weights_values)\n",
    "    # weights_values = [10,3,2,2.3]\n",
    "    print(weights_values)\n",
    "    print(df)\n",
    "    weighted_average = df.multiply(weights_values).sum(axis=1) / sum(weights_values)\n",
    "    print(weighted_average)\n",
    "    # Approximate the result to the closest integer\n",
    "    rounded_weighted_average = weighted_average.round().astype(int)\n",
    "\n",
    "    print(rounded_weighted_average)\n",
    "\n",
    "    # print(rounded_average)\n",
    "\n",
    "    df['results'] = rounded_weighted_average\n",
    "    print(df)\n",
    " \n",
    "    pred_label = df ['results'].values\n",
    "\n",
    "    # pred_label = df ['ensemble'].values\n",
    "    # df.pop('ensemble')\n",
    "    df.pop('results')\n",
    "\n",
    "    # df.pop('column_sums')\n",
    "\n",
    "    #testing metrics def\n",
    "    name = 'weighed_avg'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging  with DT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0   1.0   2.0\n",
      "0.0  20.0   0.0   0.0\n",
      "1.0   1.0  18.0   0.0\n",
      "2.0   0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9818181818181818\n",
      "Precision total:  0.9841269841269842\n",
      "Recall total:  0.9824561403508771\n",
      "F1 total:  0.9828609096901779\n",
      "BACC total:  0.9824561403508771\n",
      "MCC total:  0.973081241361401\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "start = time.time()\n",
    "base_classifier = DecisionTreeClassifier(random_state=42)\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_dt'\n",
    "pred_label = y_pred\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging  with SVM\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "     0.0   1.0   2.0\n",
      "0.0  0.0  20.0   0.0\n",
      "1.0  1.0  18.0   0.0\n",
      "2.0  0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.6181818181818182\n",
      "Precision total:  0.49122807017543857\n",
      "Recall total:  0.6491228070175438\n",
      "F1 total:  0.543859649122807\n",
      "BACC total:  0.6491228070175438\n",
      "MCC total:  0.5347990635708555\n"
     ]
    }
   ],
   "source": [
    "## bagging  with SVM\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "# Instantiate the SGDClassifier with additional hyperparameters\n",
    "svm_01 = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "\n",
    "# # Define the base classifier (Decision Tree in this case)\n",
    "base_classifier = svm_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_svm'\n",
    "pred_label = y_pred\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging with DNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from tensorflow.keras.models import Sequential\n",
    "# from tensorflow.keras.layers import Dense\n",
    "\n",
    "# #Model Parameters\n",
    "# dropout_rate = 0.2\n",
    "# nodes = 3\n",
    "# out_layer = 5\n",
    "# optimizer='adam'\n",
    "# loss='sparse_categorical_crossentropy'\n",
    "# epochs=100\n",
    "# batch_size=128\n",
    "\n",
    "\n",
    "# num_columns = X_train_01.shape[1]\n",
    "\n",
    "# dnn_01 = tf.keras.Sequential()\n",
    "\n",
    "# # Input layer\n",
    "# dnn_01.add(tf.keras.Input(shape=(num_columns,)))\n",
    "\n",
    "# # Dense layers with dropout\n",
    "# dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# # Output layer\n",
    "# # dnn_01.add(tf.keras.layers.Dense(out_layer))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "\n",
    "\n",
    "# dnn_01.compile(optimizer=optimizer, loss=loss,metrics=['accuracy'])\n",
    "\n",
    "# base_classifier = dnn_01\n",
    "\n",
    "# # Define the BaggingClassifier\n",
    "# bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# # Train the BaggingClassifier\n",
    "# bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# # Make predictions on the test set\n",
    "# y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# # Evaluate accuracy\n",
    "# # accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# # print(f'Accuracy: {accuracy}')\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('Bagging with DNN', file = f)\n",
    "\n",
    "\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "# print('CONFUSION MATRIX')\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "# pred_label = y_pred\n",
    "\n",
    "# confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "# all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "# z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "# rows, cols = confusion_matrix.shape\n",
    "# z[:rows, :cols] = confusion_matrix\n",
    "# confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "# # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "# # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "# print(confusion_matrix)\n",
    "# with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "# FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "# FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "# TP = np.diag(confusion_matrix)\n",
    "# TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "# TP_total = sum(TP)\n",
    "# TN_total = sum(TN)\n",
    "# FP_total = sum(FP)\n",
    "# FN_total = sum(FN)\n",
    "\n",
    "# TP_total = np.array(TP_total,dtype=np.float64)\n",
    "# TN_total = np.array(TN_total,dtype=np.float64)\n",
    "# FP_total = np.array(FP_total,dtype=np.float64)\n",
    "# FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "# #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "# print('METRICS')\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "# Acc = accuracy_score(y_test_01, pred_label)\n",
    "# Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "# Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "# F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "# BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "# MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "\n",
    "# bag_dnn_acc_01 = Acc\n",
    "# bag_dnn_pre_01 = Precision\n",
    "# bag_dnn_rec_01 = Recall\n",
    "# bag_dnn_f1_01 = F1\n",
    "# bag_dnn_bacc_01 = BACC\n",
    "# bag_dnn_mcc_01 = MCC\n",
    "# # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "# print('Accuracy total: ', Acc)\n",
    "# print('Precision total: ', Precision )\n",
    "# print('Recall total: ', Recall )\n",
    "# print('F1 total: ', F1 )\n",
    "# print('BACC total: ', BACC)\n",
    "# print('MCC total: ', MCC)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging with MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0   1.0   2.0\n",
      "0.0  20.0   0.0   0.0\n",
      "1.0   2.0  17.0   0.0\n",
      "2.0   0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9636363636363636\n",
      "Precision total:  0.9696969696969697\n",
      "Recall total:  0.9649122807017544\n",
      "F1 total:  0.9656084656084656\n",
      "BACC total:  0.9649122807017544\n",
      "MCC total:  0.9470572003091526\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n"
     ]
    }
   ],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "start = time.time()\n",
    "\n",
    "# create MLPClassifier instance\n",
    "mlp_01 = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "\n",
    "base_classifier = mlp_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_mlp'\n",
    "pred_label = y_pred\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging knn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0   1.0   2.0\n",
      "0.0  20.0   0.0   0.0\n",
      "1.0   0.0  19.0   0.0\n",
      "2.0   0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  1.0\n",
      "Precision total:  1.0\n",
      "Recall total:  1.0\n",
      "F1 total:  1.0\n",
      "BACC total:  1.0\n",
      "MCC total:  1.0\n"
     ]
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_01=KNeighborsClassifier(n_neighbors = 5)\n",
    "start = time.time()\n",
    "\n",
    "base_classifier = knn_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_knn'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging LogRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining baggin Logistic Regression Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0   1.0   2.0\n",
      "0.0  20.0   0.0   0.0\n",
      "1.0   2.0  17.0   0.0\n",
      "2.0   0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9636363636363636\n",
      "Precision total:  0.9696969696969697\n",
      "Recall total:  0.9649122807017544\n",
      "F1 total:  0.9656084656084656\n",
      "BACC total:  0.9649122807017544\n",
      "MCC total:  0.9470572003091526\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "start = time.time()\n",
    "\n",
    "#Logistic Regression\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining baggin Logistic Regression Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "logreg_01 = LogisticRegression()\n",
    "\n",
    "\n",
    "base_classifier = logreg_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_lr'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging ADA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0   1.0   2.0\n",
      "0.0  20.0   0.0   0.0\n",
      "1.0   1.0  18.0   0.0\n",
      "2.0   0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9818181818181818\n",
      "Precision total:  0.9841269841269842\n",
      "Recall total:  0.9824561403508771\n",
      "F1 total:  0.9828609096901779\n",
      "BACC total:  0.9824561403508771\n",
      "MCC total:  0.973081241361401\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "import time\n",
    "ada = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "\n",
    "base_classifier = ada\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_ada'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging CAT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 0.9838169\ttotal: 48.1ms\tremaining: 4.77s\n",
      "1:\tlearn: 0.8709601\ttotal: 50.1ms\tremaining: 2.45s\n",
      "2:\tlearn: 0.7784011\ttotal: 51ms\tremaining: 1.65s\n",
      "3:\tlearn: 0.6987730\ttotal: 52ms\tremaining: 1.25s\n",
      "4:\tlearn: 0.6338738\ttotal: 53.2ms\tremaining: 1.01s\n",
      "5:\tlearn: 0.5890662\ttotal: 54.3ms\tremaining: 851ms\n",
      "6:\tlearn: 0.5453196\ttotal: 55.6ms\tremaining: 738ms\n",
      "7:\tlearn: 0.5033941\ttotal: 56.6ms\tremaining: 651ms\n",
      "8:\tlearn: 0.4674938\ttotal: 57.7ms\tremaining: 584ms\n",
      "9:\tlearn: 0.4313788\ttotal: 58.7ms\tremaining: 528ms\n",
      "10:\tlearn: 0.4003634\ttotal: 60ms\tremaining: 485ms\n",
      "11:\tlearn: 0.3753104\ttotal: 61.4ms\tremaining: 450ms\n",
      "12:\tlearn: 0.3487135\ttotal: 62.9ms\tremaining: 421ms\n",
      "13:\tlearn: 0.3275741\ttotal: 64.1ms\tremaining: 394ms\n",
      "14:\tlearn: 0.3086219\ttotal: 65.3ms\tremaining: 370ms\n",
      "15:\tlearn: 0.2905928\ttotal: 66.4ms\tremaining: 349ms\n",
      "16:\tlearn: 0.2720239\ttotal: 67.5ms\tremaining: 330ms\n",
      "17:\tlearn: 0.2569596\ttotal: 68.8ms\tremaining: 313ms\n",
      "18:\tlearn: 0.2402750\ttotal: 69.9ms\tremaining: 298ms\n",
      "19:\tlearn: 0.2276040\ttotal: 71.2ms\tremaining: 285ms\n",
      "20:\tlearn: 0.2147927\ttotal: 72.5ms\tremaining: 273ms\n",
      "21:\tlearn: 0.2026011\ttotal: 73.7ms\tremaining: 261ms\n",
      "22:\tlearn: 0.1937194\ttotal: 75ms\tremaining: 251ms\n",
      "23:\tlearn: 0.1834228\ttotal: 76.1ms\tremaining: 241ms\n",
      "24:\tlearn: 0.1751489\ttotal: 77.3ms\tremaining: 232ms\n",
      "25:\tlearn: 0.1663391\ttotal: 78.5ms\tremaining: 223ms\n",
      "26:\tlearn: 0.1585618\ttotal: 79.7ms\tremaining: 216ms\n",
      "27:\tlearn: 0.1530102\ttotal: 80.8ms\tremaining: 208ms\n",
      "28:\tlearn: 0.1463045\ttotal: 81.7ms\tremaining: 200ms\n",
      "29:\tlearn: 0.1396589\ttotal: 82.5ms\tremaining: 192ms\n",
      "30:\tlearn: 0.1335783\ttotal: 83.2ms\tremaining: 185ms\n",
      "31:\tlearn: 0.1280202\ttotal: 84.2ms\tremaining: 179ms\n",
      "32:\tlearn: 0.1227231\ttotal: 85.1ms\tremaining: 173ms\n",
      "33:\tlearn: 0.1163248\ttotal: 85.9ms\tremaining: 167ms\n",
      "34:\tlearn: 0.1118465\ttotal: 86.7ms\tremaining: 161ms\n",
      "35:\tlearn: 0.1080030\ttotal: 87.7ms\tremaining: 156ms\n",
      "36:\tlearn: 0.1047212\ttotal: 88.7ms\tremaining: 151ms\n",
      "37:\tlearn: 0.1003323\ttotal: 89.6ms\tremaining: 146ms\n",
      "38:\tlearn: 0.0965006\ttotal: 90.5ms\tremaining: 141ms\n",
      "39:\tlearn: 0.0926506\ttotal: 91.2ms\tremaining: 137ms\n",
      "40:\tlearn: 0.0906097\ttotal: 91.5ms\tremaining: 132ms\n",
      "41:\tlearn: 0.0874393\ttotal: 92.2ms\tremaining: 127ms\n",
      "42:\tlearn: 0.0837924\ttotal: 93ms\tremaining: 123ms\n",
      "43:\tlearn: 0.0814871\ttotal: 93.7ms\tremaining: 119ms\n",
      "44:\tlearn: 0.0785577\ttotal: 94.4ms\tremaining: 115ms\n",
      "45:\tlearn: 0.0760136\ttotal: 95.2ms\tremaining: 112ms\n",
      "46:\tlearn: 0.0734478\ttotal: 95.8ms\tremaining: 108ms\n",
      "47:\tlearn: 0.0710189\ttotal: 96.5ms\tremaining: 105ms\n",
      "48:\tlearn: 0.0691146\ttotal: 97.2ms\tremaining: 101ms\n",
      "49:\tlearn: 0.0671774\ttotal: 97.9ms\tremaining: 97.9ms\n",
      "50:\tlearn: 0.0652778\ttotal: 98.8ms\tremaining: 94.9ms\n",
      "51:\tlearn: 0.0633899\ttotal: 99.5ms\tremaining: 91.8ms\n",
      "52:\tlearn: 0.0616760\ttotal: 100ms\tremaining: 88.8ms\n",
      "53:\tlearn: 0.0600575\ttotal: 101ms\tremaining: 85.8ms\n",
      "54:\tlearn: 0.0583515\ttotal: 101ms\tremaining: 83ms\n",
      "55:\tlearn: 0.0569643\ttotal: 102ms\tremaining: 80ms\n",
      "56:\tlearn: 0.0553872\ttotal: 102ms\tremaining: 77.3ms\n",
      "57:\tlearn: 0.0540484\ttotal: 103ms\tremaining: 74.7ms\n",
      "58:\tlearn: 0.0524516\ttotal: 104ms\tremaining: 72.2ms\n",
      "59:\tlearn: 0.0509107\ttotal: 105ms\tremaining: 69.7ms\n",
      "60:\tlearn: 0.0497531\ttotal: 105ms\tremaining: 67.4ms\n",
      "61:\tlearn: 0.0485198\ttotal: 106ms\tremaining: 64.9ms\n",
      "62:\tlearn: 0.0470643\ttotal: 107ms\tremaining: 62.6ms\n",
      "63:\tlearn: 0.0461288\ttotal: 107ms\tremaining: 60.3ms\n",
      "64:\tlearn: 0.0449904\ttotal: 108ms\tremaining: 58.1ms\n",
      "65:\tlearn: 0.0438530\ttotal: 109ms\tremaining: 56.2ms\n",
      "66:\tlearn: 0.0430348\ttotal: 110ms\tremaining: 54.2ms\n",
      "67:\tlearn: 0.0421650\ttotal: 111ms\tremaining: 52.1ms\n",
      "68:\tlearn: 0.0411723\ttotal: 111ms\tremaining: 50ms\n",
      "69:\tlearn: 0.0401137\ttotal: 112ms\tremaining: 48ms\n",
      "70:\tlearn: 0.0393050\ttotal: 113ms\tremaining: 46ms\n",
      "71:\tlearn: 0.0386699\ttotal: 113ms\tremaining: 44.1ms\n",
      "72:\tlearn: 0.0380653\ttotal: 114ms\tremaining: 42.1ms\n",
      "73:\tlearn: 0.0373579\ttotal: 115ms\tremaining: 40.3ms\n",
      "74:\tlearn: 0.0364391\ttotal: 116ms\tremaining: 38.5ms\n",
      "75:\tlearn: 0.0357641\ttotal: 116ms\tremaining: 36.7ms\n",
      "76:\tlearn: 0.0351766\ttotal: 117ms\tremaining: 35ms\n",
      "77:\tlearn: 0.0344162\ttotal: 118ms\tremaining: 33.2ms\n",
      "78:\tlearn: 0.0338202\ttotal: 118ms\tremaining: 31.4ms\n",
      "79:\tlearn: 0.0332588\ttotal: 119ms\tremaining: 29.7ms\n",
      "80:\tlearn: 0.0326858\ttotal: 120ms\tremaining: 28ms\n",
      "81:\tlearn: 0.0320905\ttotal: 120ms\tremaining: 26.4ms\n",
      "82:\tlearn: 0.0316574\ttotal: 121ms\tremaining: 24.8ms\n",
      "83:\tlearn: 0.0311329\ttotal: 122ms\tremaining: 23.2ms\n",
      "84:\tlearn: 0.0306131\ttotal: 122ms\tremaining: 21.6ms\n",
      "85:\tlearn: 0.0301369\ttotal: 123ms\tremaining: 20.1ms\n",
      "86:\tlearn: 0.0296809\ttotal: 124ms\tremaining: 18.5ms\n",
      "87:\tlearn: 0.0291865\ttotal: 125ms\tremaining: 17ms\n",
      "88:\tlearn: 0.0286271\ttotal: 126ms\tremaining: 15.6ms\n",
      "89:\tlearn: 0.0281905\ttotal: 127ms\tremaining: 14.1ms\n",
      "90:\tlearn: 0.0276732\ttotal: 128ms\tremaining: 12.6ms\n",
      "91:\tlearn: 0.0272601\ttotal: 128ms\tremaining: 11.2ms\n",
      "92:\tlearn: 0.0269287\ttotal: 129ms\tremaining: 9.72ms\n",
      "93:\tlearn: 0.0265465\ttotal: 130ms\tremaining: 8.29ms\n",
      "94:\tlearn: 0.0261755\ttotal: 130ms\tremaining: 6.87ms\n",
      "95:\tlearn: 0.0258155\ttotal: 131ms\tremaining: 5.46ms\n",
      "96:\tlearn: 0.0254444\ttotal: 132ms\tremaining: 4.07ms\n",
      "97:\tlearn: 0.0250954\ttotal: 132ms\tremaining: 2.7ms\n",
      "98:\tlearn: 0.0249583\ttotal: 133ms\tremaining: 1.34ms\n",
      "99:\tlearn: 0.0245875\ttotal: 134ms\tremaining: 0us\n",
      "0:\tlearn: 0.9854383\ttotal: 1.13ms\tremaining: 112ms\n",
      "1:\tlearn: 0.8923507\ttotal: 2.5ms\tremaining: 123ms\n",
      "2:\tlearn: 0.7991788\ttotal: 3.22ms\tremaining: 104ms\n",
      "3:\tlearn: 0.7277186\ttotal: 3.74ms\tremaining: 89.8ms\n",
      "4:\tlearn: 0.6759610\ttotal: 4.58ms\tremaining: 87ms\n",
      "5:\tlearn: 0.6151415\ttotal: 5.96ms\tremaining: 93.3ms\n",
      "6:\tlearn: 0.5656795\ttotal: 7.42ms\tremaining: 98.5ms\n",
      "7:\tlearn: 0.5248781\ttotal: 8.75ms\tremaining: 101ms\n",
      "8:\tlearn: 0.4858612\ttotal: 9.78ms\tremaining: 98.9ms\n",
      "9:\tlearn: 0.4559682\ttotal: 11.3ms\tremaining: 102ms\n",
      "10:\tlearn: 0.4235965\ttotal: 12.4ms\tremaining: 100ms\n",
      "11:\tlearn: 0.4000556\ttotal: 13.4ms\tremaining: 98.3ms\n",
      "12:\tlearn: 0.3714147\ttotal: 14.3ms\tremaining: 95.7ms\n",
      "13:\tlearn: 0.3523345\ttotal: 15.3ms\tremaining: 93.9ms\n",
      "14:\tlearn: 0.3318771\ttotal: 16.3ms\tremaining: 92.2ms\n",
      "15:\tlearn: 0.3115442\ttotal: 17.2ms\tremaining: 90.1ms\n",
      "16:\tlearn: 0.2936531\ttotal: 18ms\tremaining: 88ms\n",
      "17:\tlearn: 0.2757264\ttotal: 19ms\tremaining: 86.4ms\n",
      "18:\tlearn: 0.2596228\ttotal: 19.8ms\tremaining: 84.4ms\n",
      "19:\tlearn: 0.2442931\ttotal: 20.5ms\tremaining: 82.1ms\n",
      "20:\tlearn: 0.2307895\ttotal: 21.2ms\tremaining: 79.7ms\n",
      "21:\tlearn: 0.2205247\ttotal: 22ms\tremaining: 77.9ms\n",
      "22:\tlearn: 0.2098437\ttotal: 22.8ms\tremaining: 76.4ms\n",
      "23:\tlearn: 0.2000924\ttotal: 24.3ms\tremaining: 76.9ms\n",
      "24:\tlearn: 0.1912724\ttotal: 25.1ms\tremaining: 75.4ms\n",
      "25:\tlearn: 0.1818158\ttotal: 25.9ms\tremaining: 73.8ms\n",
      "26:\tlearn: 0.1746341\ttotal: 27ms\tremaining: 73ms\n",
      "27:\tlearn: 0.1674209\ttotal: 27.8ms\tremaining: 71.5ms\n",
      "28:\tlearn: 0.1594382\ttotal: 28.5ms\tremaining: 69.7ms\n",
      "29:\tlearn: 0.1528696\ttotal: 29.2ms\tremaining: 68.2ms\n",
      "30:\tlearn: 0.1453201\ttotal: 30ms\tremaining: 66.8ms\n",
      "31:\tlearn: 0.1390218\ttotal: 30.8ms\tremaining: 65.5ms\n",
      "32:\tlearn: 0.1337715\ttotal: 31.6ms\tremaining: 64.1ms\n",
      "33:\tlearn: 0.1287975\ttotal: 32.3ms\tremaining: 62.7ms\n",
      "34:\tlearn: 0.1243639\ttotal: 33ms\tremaining: 61.2ms\n",
      "35:\tlearn: 0.1205929\ttotal: 33.6ms\tremaining: 59.7ms\n",
      "36:\tlearn: 0.1164498\ttotal: 34.3ms\tremaining: 58.4ms\n",
      "37:\tlearn: 0.1122923\ttotal: 35ms\tremaining: 57.1ms\n",
      "38:\tlearn: 0.1082190\ttotal: 35.4ms\tremaining: 55.4ms\n",
      "39:\tlearn: 0.1043241\ttotal: 36.2ms\tremaining: 54.4ms\n",
      "40:\tlearn: 0.1012151\ttotal: 37.1ms\tremaining: 53.3ms\n",
      "41:\tlearn: 0.0986924\ttotal: 37.8ms\tremaining: 52.2ms\n",
      "42:\tlearn: 0.0963276\ttotal: 38.5ms\tremaining: 51.1ms\n",
      "43:\tlearn: 0.0935309\ttotal: 39.3ms\tremaining: 50ms\n",
      "44:\tlearn: 0.0906478\ttotal: 40.4ms\tremaining: 49.4ms\n",
      "45:\tlearn: 0.0878397\ttotal: 41.3ms\tremaining: 48.5ms\n",
      "46:\tlearn: 0.0850109\ttotal: 42ms\tremaining: 47.3ms\n",
      "47:\tlearn: 0.0826114\ttotal: 42.7ms\tremaining: 46.2ms\n",
      "48:\tlearn: 0.0800164\ttotal: 43.4ms\tremaining: 45.2ms\n",
      "49:\tlearn: 0.0778601\ttotal: 44.6ms\tremaining: 44.6ms\n",
      "50:\tlearn: 0.0758696\ttotal: 45.5ms\tremaining: 43.7ms\n",
      "51:\tlearn: 0.0739578\ttotal: 46.2ms\tremaining: 42.6ms\n",
      "52:\tlearn: 0.0719587\ttotal: 46.9ms\tremaining: 41.6ms\n",
      "53:\tlearn: 0.0698482\ttotal: 47.6ms\tremaining: 40.5ms\n",
      "54:\tlearn: 0.0683377\ttotal: 48.3ms\tremaining: 39.6ms\n",
      "55:\tlearn: 0.0669318\ttotal: 49.1ms\tremaining: 38.6ms\n",
      "56:\tlearn: 0.0650981\ttotal: 49.8ms\tremaining: 37.5ms\n",
      "57:\tlearn: 0.0638164\ttotal: 50.5ms\tremaining: 36.6ms\n",
      "58:\tlearn: 0.0625050\ttotal: 51.1ms\tremaining: 35.5ms\n",
      "59:\tlearn: 0.0609753\ttotal: 51.7ms\tremaining: 34.5ms\n",
      "60:\tlearn: 0.0595678\ttotal: 52.2ms\tremaining: 33.4ms\n",
      "61:\tlearn: 0.0581614\ttotal: 53ms\tremaining: 32.5ms\n",
      "62:\tlearn: 0.0565243\ttotal: 53.7ms\tremaining: 31.5ms\n",
      "63:\tlearn: 0.0554289\ttotal: 54.3ms\tremaining: 30.6ms\n",
      "64:\tlearn: 0.0542659\ttotal: 55ms\tremaining: 29.6ms\n",
      "65:\tlearn: 0.0532203\ttotal: 55.6ms\tremaining: 28.7ms\n",
      "66:\tlearn: 0.0526502\ttotal: 56.1ms\tremaining: 27.6ms\n",
      "67:\tlearn: 0.0512458\ttotal: 57ms\tremaining: 26.8ms\n",
      "68:\tlearn: 0.0499010\ttotal: 57.8ms\tremaining: 26ms\n",
      "69:\tlearn: 0.0488380\ttotal: 58.5ms\tremaining: 25.1ms\n",
      "70:\tlearn: 0.0479262\ttotal: 59.2ms\tremaining: 24.2ms\n",
      "71:\tlearn: 0.0467120\ttotal: 59.6ms\tremaining: 23.2ms\n",
      "72:\tlearn: 0.0456813\ttotal: 60.3ms\tremaining: 22.3ms\n",
      "73:\tlearn: 0.0449057\ttotal: 61.1ms\tremaining: 21.5ms\n",
      "74:\tlearn: 0.0439207\ttotal: 61.7ms\tremaining: 20.6ms\n",
      "75:\tlearn: 0.0430185\ttotal: 62.3ms\tremaining: 19.7ms\n",
      "76:\tlearn: 0.0420782\ttotal: 63ms\tremaining: 18.8ms\n",
      "77:\tlearn: 0.0411445\ttotal: 63.7ms\tremaining: 18ms\n",
      "78:\tlearn: 0.0403928\ttotal: 64.5ms\tremaining: 17.2ms\n",
      "79:\tlearn: 0.0395669\ttotal: 65.3ms\tremaining: 16.3ms\n",
      "80:\tlearn: 0.0390516\ttotal: 65.9ms\tremaining: 15.5ms\n",
      "81:\tlearn: 0.0383268\ttotal: 66.6ms\tremaining: 14.6ms\n",
      "82:\tlearn: 0.0376553\ttotal: 67.4ms\tremaining: 13.8ms\n",
      "83:\tlearn: 0.0369725\ttotal: 68.1ms\tremaining: 13ms\n",
      "84:\tlearn: 0.0362782\ttotal: 68.7ms\tremaining: 12.1ms\n",
      "85:\tlearn: 0.0360517\ttotal: 69.4ms\tremaining: 11.3ms\n",
      "86:\tlearn: 0.0355650\ttotal: 70.1ms\tremaining: 10.5ms\n",
      "87:\tlearn: 0.0349737\ttotal: 70.8ms\tremaining: 9.65ms\n",
      "88:\tlearn: 0.0343665\ttotal: 71.4ms\tremaining: 8.83ms\n",
      "89:\tlearn: 0.0337960\ttotal: 72.1ms\tremaining: 8.01ms\n",
      "90:\tlearn: 0.0332478\ttotal: 72.7ms\tremaining: 7.19ms\n",
      "91:\tlearn: 0.0327623\ttotal: 73.6ms\tremaining: 6.4ms\n",
      "92:\tlearn: 0.0322650\ttotal: 74.5ms\tremaining: 5.61ms\n",
      "93:\tlearn: 0.0317556\ttotal: 75.3ms\tremaining: 4.8ms\n",
      "94:\tlearn: 0.0312908\ttotal: 75.9ms\tremaining: 3.99ms\n",
      "95:\tlearn: 0.0309308\ttotal: 76.5ms\tremaining: 3.19ms\n",
      "96:\tlearn: 0.0305928\ttotal: 77.1ms\tremaining: 2.38ms\n",
      "97:\tlearn: 0.0301292\ttotal: 77.7ms\tremaining: 1.59ms\n",
      "98:\tlearn: 0.0297440\ttotal: 78.4ms\tremaining: 791us\n",
      "99:\tlearn: 0.0293223\ttotal: 79ms\tremaining: 0us\n",
      "0:\tlearn: 0.9745760\ttotal: 1.39ms\tremaining: 137ms\n",
      "1:\tlearn: 0.8706957\ttotal: 2.96ms\tremaining: 145ms\n",
      "2:\tlearn: 0.7767556\ttotal: 3.95ms\tremaining: 128ms\n",
      "3:\tlearn: 0.6970738\ttotal: 10ms\tremaining: 241ms\n",
      "4:\tlearn: 0.6381105\ttotal: 12ms\tremaining: 227ms\n",
      "5:\tlearn: 0.5787158\ttotal: 14.7ms\tremaining: 230ms\n",
      "6:\tlearn: 0.5264752\ttotal: 15.8ms\tremaining: 210ms\n",
      "7:\tlearn: 0.4836064\ttotal: 17ms\tremaining: 195ms\n",
      "8:\tlearn: 0.4457529\ttotal: 18.1ms\tremaining: 183ms\n",
      "9:\tlearn: 0.4156445\ttotal: 19.2ms\tremaining: 173ms\n",
      "10:\tlearn: 0.3876272\ttotal: 20.3ms\tremaining: 164ms\n",
      "11:\tlearn: 0.3613305\ttotal: 21.2ms\tremaining: 156ms\n",
      "12:\tlearn: 0.3347780\ttotal: 22.1ms\tremaining: 148ms\n",
      "13:\tlearn: 0.3136483\ttotal: 23ms\tremaining: 142ms\n",
      "14:\tlearn: 0.2904980\ttotal: 23.7ms\tremaining: 134ms\n",
      "15:\tlearn: 0.2717642\ttotal: 24.5ms\tremaining: 128ms\n",
      "16:\tlearn: 0.2550343\ttotal: 25.4ms\tremaining: 124ms\n",
      "17:\tlearn: 0.2369965\ttotal: 26ms\tremaining: 118ms\n",
      "18:\tlearn: 0.2232611\ttotal: 26.9ms\tremaining: 115ms\n",
      "19:\tlearn: 0.2116699\ttotal: 27.8ms\tremaining: 111ms\n",
      "20:\tlearn: 0.1991651\ttotal: 28.9ms\tremaining: 109ms\n",
      "21:\tlearn: 0.1898077\ttotal: 29.8ms\tremaining: 105ms\n",
      "22:\tlearn: 0.1799269\ttotal: 30.6ms\tremaining: 102ms\n",
      "23:\tlearn: 0.1709869\ttotal: 31.4ms\tremaining: 99.4ms\n",
      "24:\tlearn: 0.1619557\ttotal: 32.1ms\tremaining: 96.3ms\n",
      "25:\tlearn: 0.1532292\ttotal: 32.6ms\tremaining: 92.9ms\n",
      "26:\tlearn: 0.1468970\ttotal: 33.4ms\tremaining: 90.3ms\n",
      "27:\tlearn: 0.1405287\ttotal: 34.2ms\tremaining: 87.9ms\n",
      "28:\tlearn: 0.1333171\ttotal: 34.9ms\tremaining: 85.4ms\n",
      "29:\tlearn: 0.1275346\ttotal: 35.7ms\tremaining: 83.3ms\n",
      "30:\tlearn: 0.1229995\ttotal: 36.4ms\tremaining: 80.9ms\n",
      "31:\tlearn: 0.1167463\ttotal: 36.9ms\tremaining: 78.4ms\n",
      "32:\tlearn: 0.1121136\ttotal: 37.9ms\tremaining: 76.9ms\n",
      "33:\tlearn: 0.1068509\ttotal: 38.4ms\tremaining: 74.5ms\n",
      "34:\tlearn: 0.1023949\ttotal: 39.2ms\tremaining: 72.7ms\n",
      "35:\tlearn: 0.0981202\ttotal: 40ms\tremaining: 71.1ms\n",
      "36:\tlearn: 0.0948183\ttotal: 40.8ms\tremaining: 69.5ms\n",
      "37:\tlearn: 0.0913284\ttotal: 41.6ms\tremaining: 67.8ms\n",
      "38:\tlearn: 0.0877634\ttotal: 42.5ms\tremaining: 66.5ms\n",
      "39:\tlearn: 0.0846134\ttotal: 43.3ms\tremaining: 65ms\n",
      "40:\tlearn: 0.0813523\ttotal: 44.1ms\tremaining: 63.4ms\n",
      "41:\tlearn: 0.0786053\ttotal: 44.9ms\tremaining: 62ms\n",
      "42:\tlearn: 0.0763207\ttotal: 45.4ms\tremaining: 60.1ms\n",
      "43:\tlearn: 0.0740744\ttotal: 46.2ms\tremaining: 58.8ms\n",
      "44:\tlearn: 0.0720847\ttotal: 47ms\tremaining: 57.4ms\n",
      "45:\tlearn: 0.0699654\ttotal: 47.8ms\tremaining: 56.1ms\n",
      "46:\tlearn: 0.0678171\ttotal: 48.5ms\tremaining: 54.7ms\n",
      "47:\tlearn: 0.0660068\ttotal: 49.3ms\tremaining: 53.4ms\n",
      "48:\tlearn: 0.0639959\ttotal: 49.9ms\tremaining: 51.9ms\n",
      "49:\tlearn: 0.0623376\ttotal: 50.5ms\tremaining: 50.5ms\n",
      "50:\tlearn: 0.0606400\ttotal: 51.2ms\tremaining: 49.2ms\n",
      "51:\tlearn: 0.0588581\ttotal: 51.8ms\tremaining: 47.8ms\n",
      "52:\tlearn: 0.0573974\ttotal: 52.4ms\tremaining: 46.5ms\n",
      "53:\tlearn: 0.0560152\ttotal: 53.1ms\tremaining: 45.3ms\n",
      "54:\tlearn: 0.0545982\ttotal: 53.8ms\tremaining: 44ms\n",
      "55:\tlearn: 0.0532208\ttotal: 54.4ms\tremaining: 42.8ms\n",
      "56:\tlearn: 0.0521271\ttotal: 55.4ms\tremaining: 41.8ms\n",
      "57:\tlearn: 0.0509710\ttotal: 56.1ms\tremaining: 40.6ms\n",
      "58:\tlearn: 0.0497155\ttotal: 56.8ms\tremaining: 39.5ms\n",
      "59:\tlearn: 0.0484567\ttotal: 57.6ms\tremaining: 38.4ms\n",
      "60:\tlearn: 0.0473025\ttotal: 58.3ms\tremaining: 37.3ms\n",
      "61:\tlearn: 0.0461578\ttotal: 58.9ms\tremaining: 36.1ms\n",
      "62:\tlearn: 0.0452828\ttotal: 59.6ms\tremaining: 35ms\n",
      "63:\tlearn: 0.0442580\ttotal: 60.3ms\tremaining: 33.9ms\n",
      "64:\tlearn: 0.0432014\ttotal: 60.9ms\tremaining: 32.8ms\n",
      "65:\tlearn: 0.0421891\ttotal: 61.3ms\tremaining: 31.6ms\n",
      "66:\tlearn: 0.0412498\ttotal: 62ms\tremaining: 30.5ms\n",
      "67:\tlearn: 0.0403365\ttotal: 62.6ms\tremaining: 29.4ms\n",
      "68:\tlearn: 0.0393084\ttotal: 63.1ms\tremaining: 28.3ms\n",
      "69:\tlearn: 0.0385429\ttotal: 63.5ms\tremaining: 27.2ms\n",
      "70:\tlearn: 0.0378438\ttotal: 64.2ms\tremaining: 26.2ms\n",
      "71:\tlearn: 0.0371602\ttotal: 64.9ms\tremaining: 25.2ms\n",
      "72:\tlearn: 0.0364105\ttotal: 65.6ms\tremaining: 24.3ms\n",
      "73:\tlearn: 0.0356595\ttotal: 66.3ms\tremaining: 23.3ms\n",
      "74:\tlearn: 0.0350686\ttotal: 66.9ms\tremaining: 22.3ms\n",
      "75:\tlearn: 0.0344203\ttotal: 67.5ms\tremaining: 21.3ms\n",
      "76:\tlearn: 0.0337120\ttotal: 68.1ms\tremaining: 20.3ms\n",
      "77:\tlearn: 0.0330988\ttotal: 68.7ms\tremaining: 19.4ms\n",
      "78:\tlearn: 0.0325168\ttotal: 69.2ms\tremaining: 18.4ms\n",
      "79:\tlearn: 0.0319123\ttotal: 69.8ms\tremaining: 17.5ms\n",
      "80:\tlearn: 0.0314745\ttotal: 70.4ms\tremaining: 16.5ms\n",
      "81:\tlearn: 0.0307587\ttotal: 71ms\tremaining: 15.6ms\n",
      "82:\tlearn: 0.0302854\ttotal: 71.6ms\tremaining: 14.7ms\n",
      "83:\tlearn: 0.0297248\ttotal: 72.3ms\tremaining: 13.8ms\n",
      "84:\tlearn: 0.0290473\ttotal: 73ms\tremaining: 12.9ms\n",
      "85:\tlearn: 0.0285826\ttotal: 73.8ms\tremaining: 12ms\n",
      "86:\tlearn: 0.0281237\ttotal: 74.5ms\tremaining: 11.1ms\n",
      "87:\tlearn: 0.0276841\ttotal: 75.2ms\tremaining: 10.2ms\n",
      "88:\tlearn: 0.0272909\ttotal: 75.8ms\tremaining: 9.37ms\n",
      "89:\tlearn: 0.0269144\ttotal: 76.6ms\tremaining: 8.51ms\n",
      "90:\tlearn: 0.0265222\ttotal: 77.2ms\tremaining: 7.64ms\n",
      "91:\tlearn: 0.0261400\ttotal: 78ms\tremaining: 6.78ms\n",
      "92:\tlearn: 0.0257133\ttotal: 78.7ms\tremaining: 5.92ms\n",
      "93:\tlearn: 0.0253814\ttotal: 79.5ms\tremaining: 5.08ms\n",
      "94:\tlearn: 0.0250282\ttotal: 80.3ms\tremaining: 4.22ms\n",
      "95:\tlearn: 0.0246327\ttotal: 80.9ms\tremaining: 3.37ms\n",
      "96:\tlearn: 0.0242930\ttotal: 81.5ms\tremaining: 2.52ms\n",
      "97:\tlearn: 0.0239190\ttotal: 82.1ms\tremaining: 1.68ms\n",
      "98:\tlearn: 0.0235827\ttotal: 82.8ms\tremaining: 836us\n",
      "99:\tlearn: 0.0232723\ttotal: 83.5ms\tremaining: 0us\n",
      "0:\tlearn: 0.9819302\ttotal: 966us\tremaining: 95.7ms\n",
      "1:\tlearn: 0.8900005\ttotal: 2.12ms\tremaining: 104ms\n",
      "2:\tlearn: 0.8062534\ttotal: 3.02ms\tremaining: 97.7ms\n",
      "3:\tlearn: 0.7338769\ttotal: 4.02ms\tremaining: 96.4ms\n",
      "4:\tlearn: 0.6690205\ttotal: 4.9ms\tremaining: 93.1ms\n",
      "5:\tlearn: 0.6097473\ttotal: 6.09ms\tremaining: 95.4ms\n",
      "6:\tlearn: 0.5624552\ttotal: 7.15ms\tremaining: 95ms\n",
      "7:\tlearn: 0.5176559\ttotal: 8.43ms\tremaining: 96.9ms\n",
      "8:\tlearn: 0.4765505\ttotal: 9.66ms\tremaining: 97.7ms\n",
      "9:\tlearn: 0.4448604\ttotal: 10.9ms\tremaining: 97.9ms\n",
      "10:\tlearn: 0.4163887\ttotal: 12.1ms\tremaining: 98.1ms\n",
      "11:\tlearn: 0.3879717\ttotal: 13.4ms\tremaining: 98.1ms\n",
      "12:\tlearn: 0.3635391\ttotal: 14.5ms\tremaining: 96.7ms\n",
      "13:\tlearn: 0.3400222\ttotal: 15.6ms\tremaining: 95.7ms\n",
      "14:\tlearn: 0.3186500\ttotal: 16.7ms\tremaining: 94.9ms\n",
      "15:\tlearn: 0.3001103\ttotal: 17.7ms\tremaining: 93ms\n",
      "16:\tlearn: 0.2824827\ttotal: 18.7ms\tremaining: 91.5ms\n",
      "17:\tlearn: 0.2671807\ttotal: 19.8ms\tremaining: 90.3ms\n",
      "18:\tlearn: 0.2527305\ttotal: 20.9ms\tremaining: 89.1ms\n",
      "19:\tlearn: 0.2386335\ttotal: 22ms\tremaining: 87.8ms\n",
      "20:\tlearn: 0.2283464\ttotal: 23.3ms\tremaining: 87.5ms\n",
      "21:\tlearn: 0.2158597\ttotal: 24.5ms\tremaining: 86.9ms\n",
      "22:\tlearn: 0.2036350\ttotal: 25.7ms\tremaining: 85.9ms\n",
      "23:\tlearn: 0.1945901\ttotal: 26.9ms\tremaining: 85.1ms\n",
      "24:\tlearn: 0.1866105\ttotal: 27.9ms\tremaining: 83.6ms\n",
      "25:\tlearn: 0.1794400\ttotal: 28.8ms\tremaining: 82.1ms\n",
      "26:\tlearn: 0.1717251\ttotal: 29.9ms\tremaining: 80.7ms\n",
      "27:\tlearn: 0.1638761\ttotal: 30.8ms\tremaining: 79.2ms\n",
      "28:\tlearn: 0.1568032\ttotal: 31.8ms\tremaining: 77.8ms\n",
      "29:\tlearn: 0.1504393\ttotal: 32.7ms\tremaining: 76.3ms\n",
      "30:\tlearn: 0.1451506\ttotal: 33.8ms\tremaining: 75.2ms\n",
      "31:\tlearn: 0.1399814\ttotal: 34.9ms\tremaining: 74.1ms\n",
      "32:\tlearn: 0.1346792\ttotal: 36ms\tremaining: 73ms\n",
      "33:\tlearn: 0.1293850\ttotal: 36.9ms\tremaining: 71.7ms\n",
      "34:\tlearn: 0.1246305\ttotal: 37.9ms\tremaining: 70.3ms\n",
      "35:\tlearn: 0.1210845\ttotal: 38.9ms\tremaining: 69.1ms\n",
      "36:\tlearn: 0.1176671\ttotal: 39.7ms\tremaining: 67.7ms\n",
      "37:\tlearn: 0.1130762\ttotal: 40.7ms\tremaining: 66.4ms\n",
      "38:\tlearn: 0.1087440\ttotal: 41.7ms\tremaining: 65.2ms\n",
      "39:\tlearn: 0.1053069\ttotal: 42.7ms\tremaining: 64ms\n",
      "40:\tlearn: 0.1024834\ttotal: 43.6ms\tremaining: 62.8ms\n",
      "41:\tlearn: 0.0996344\ttotal: 44.8ms\tremaining: 61.9ms\n",
      "42:\tlearn: 0.0960388\ttotal: 45.7ms\tremaining: 60.6ms\n",
      "43:\tlearn: 0.0928606\ttotal: 46.6ms\tremaining: 59.3ms\n",
      "44:\tlearn: 0.0898147\ttotal: 47.4ms\tremaining: 58ms\n",
      "45:\tlearn: 0.0873472\ttotal: 48.2ms\tremaining: 56.5ms\n",
      "46:\tlearn: 0.0849212\ttotal: 49.1ms\tremaining: 55.4ms\n",
      "47:\tlearn: 0.0823840\ttotal: 50ms\tremaining: 54.2ms\n",
      "48:\tlearn: 0.0801212\ttotal: 50.9ms\tremaining: 52.9ms\n",
      "49:\tlearn: 0.0778178\ttotal: 51.7ms\tremaining: 51.7ms\n",
      "50:\tlearn: 0.0756122\ttotal: 52.5ms\tremaining: 50.5ms\n",
      "51:\tlearn: 0.0735457\ttotal: 53.5ms\tremaining: 49.3ms\n",
      "52:\tlearn: 0.0717958\ttotal: 54.2ms\tremaining: 48.1ms\n",
      "53:\tlearn: 0.0701845\ttotal: 55ms\tremaining: 46.9ms\n",
      "54:\tlearn: 0.0688152\ttotal: 55.7ms\tremaining: 45.6ms\n",
      "55:\tlearn: 0.0670679\ttotal: 56.5ms\tremaining: 44.4ms\n",
      "56:\tlearn: 0.0653710\ttotal: 57.3ms\tremaining: 43.3ms\n",
      "57:\tlearn: 0.0637563\ttotal: 58.4ms\tremaining: 42.3ms\n",
      "58:\tlearn: 0.0624025\ttotal: 59.5ms\tremaining: 41.4ms\n",
      "59:\tlearn: 0.0606743\ttotal: 60.4ms\tremaining: 40.3ms\n",
      "60:\tlearn: 0.0594229\ttotal: 61.2ms\tremaining: 39.1ms\n",
      "61:\tlearn: 0.0575242\ttotal: 62ms\tremaining: 38ms\n",
      "62:\tlearn: 0.0562065\ttotal: 62.8ms\tremaining: 36.9ms\n",
      "63:\tlearn: 0.0548659\ttotal: 63.5ms\tremaining: 35.7ms\n",
      "64:\tlearn: 0.0538906\ttotal: 64.2ms\tremaining: 34.6ms\n",
      "65:\tlearn: 0.0528082\ttotal: 65.1ms\tremaining: 33.5ms\n",
      "66:\tlearn: 0.0518690\ttotal: 65.5ms\tremaining: 32.3ms\n",
      "67:\tlearn: 0.0508291\ttotal: 66.4ms\tremaining: 31.2ms\n",
      "68:\tlearn: 0.0498899\ttotal: 67.1ms\tremaining: 30.2ms\n",
      "69:\tlearn: 0.0489887\ttotal: 67.9ms\tremaining: 29.1ms\n",
      "70:\tlearn: 0.0479650\ttotal: 68.7ms\tremaining: 28.1ms\n",
      "71:\tlearn: 0.0468300\ttotal: 69.5ms\tremaining: 27ms\n",
      "72:\tlearn: 0.0459552\ttotal: 73.2ms\tremaining: 27.1ms\n",
      "73:\tlearn: 0.0451030\ttotal: 74.3ms\tremaining: 26.1ms\n",
      "74:\tlearn: 0.0443162\ttotal: 74.9ms\tremaining: 25ms\n",
      "75:\tlearn: 0.0433946\ttotal: 76ms\tremaining: 24ms\n",
      "76:\tlearn: 0.0425863\ttotal: 76.9ms\tremaining: 23ms\n",
      "77:\tlearn: 0.0420266\ttotal: 77.6ms\tremaining: 21.9ms\n",
      "78:\tlearn: 0.0411792\ttotal: 78.4ms\tremaining: 20.8ms\n",
      "79:\tlearn: 0.0402877\ttotal: 79.1ms\tremaining: 19.8ms\n",
      "80:\tlearn: 0.0395608\ttotal: 79.9ms\tremaining: 18.7ms\n",
      "81:\tlearn: 0.0388855\ttotal: 80.8ms\tremaining: 17.7ms\n",
      "82:\tlearn: 0.0383143\ttotal: 81.5ms\tremaining: 16.7ms\n",
      "83:\tlearn: 0.0377014\ttotal: 82.5ms\tremaining: 15.7ms\n",
      "84:\tlearn: 0.0370110\ttotal: 83.5ms\tremaining: 14.7ms\n",
      "85:\tlearn: 0.0363440\ttotal: 84.5ms\tremaining: 13.8ms\n",
      "86:\tlearn: 0.0358535\ttotal: 85.4ms\tremaining: 12.8ms\n",
      "87:\tlearn: 0.0352941\ttotal: 86.3ms\tremaining: 11.8ms\n",
      "88:\tlearn: 0.0346191\ttotal: 87.3ms\tremaining: 10.8ms\n",
      "89:\tlearn: 0.0341069\ttotal: 88.3ms\tremaining: 9.81ms\n",
      "90:\tlearn: 0.0335494\ttotal: 89.3ms\tremaining: 8.83ms\n",
      "91:\tlearn: 0.0330691\ttotal: 90.2ms\tremaining: 7.85ms\n",
      "92:\tlearn: 0.0326479\ttotal: 91.1ms\tremaining: 6.85ms\n",
      "93:\tlearn: 0.0321104\ttotal: 91.8ms\tremaining: 5.86ms\n",
      "94:\tlearn: 0.0316395\ttotal: 92.6ms\tremaining: 4.88ms\n",
      "95:\tlearn: 0.0312589\ttotal: 93.4ms\tremaining: 3.89ms\n",
      "96:\tlearn: 0.0308635\ttotal: 94.7ms\tremaining: 2.93ms\n",
      "97:\tlearn: 0.0304138\ttotal: 95.8ms\tremaining: 1.96ms\n",
      "98:\tlearn: 0.0300541\ttotal: 96.7ms\tremaining: 976us\n",
      "99:\tlearn: 0.0296436\ttotal: 97.6ms\tremaining: 0us\n",
      "0:\tlearn: 0.9741750\ttotal: 936us\tremaining: 92.7ms\n",
      "1:\tlearn: 0.8742669\ttotal: 2.19ms\tremaining: 108ms\n",
      "2:\tlearn: 0.7888878\ttotal: 3.38ms\tremaining: 109ms\n",
      "3:\tlearn: 0.7144840\ttotal: 4.59ms\tremaining: 110ms\n",
      "4:\tlearn: 0.6535330\ttotal: 10.7ms\tremaining: 204ms\n",
      "5:\tlearn: 0.5979706\ttotal: 11.8ms\tremaining: 184ms\n",
      "6:\tlearn: 0.5465752\ttotal: 12.8ms\tremaining: 170ms\n",
      "7:\tlearn: 0.5061879\ttotal: 13.5ms\tremaining: 155ms\n",
      "8:\tlearn: 0.4659917\ttotal: 14.4ms\tremaining: 146ms\n",
      "9:\tlearn: 0.4284669\ttotal: 14.9ms\tremaining: 134ms\n",
      "10:\tlearn: 0.3958285\ttotal: 15.5ms\tremaining: 125ms\n",
      "11:\tlearn: 0.3730034\ttotal: 16.3ms\tremaining: 120ms\n",
      "12:\tlearn: 0.3468548\ttotal: 17.2ms\tremaining: 115ms\n",
      "13:\tlearn: 0.3294290\ttotal: 18.1ms\tremaining: 111ms\n",
      "14:\tlearn: 0.3102135\ttotal: 19.2ms\tremaining: 109ms\n",
      "15:\tlearn: 0.2917664\ttotal: 20.2ms\tremaining: 106ms\n",
      "16:\tlearn: 0.2744554\ttotal: 21.1ms\tremaining: 103ms\n",
      "17:\tlearn: 0.2575264\ttotal: 21.9ms\tremaining: 99.8ms\n",
      "18:\tlearn: 0.2405878\ttotal: 22.8ms\tremaining: 97.2ms\n",
      "19:\tlearn: 0.2272987\ttotal: 23.7ms\tremaining: 94.7ms\n",
      "20:\tlearn: 0.2157479\ttotal: 24.6ms\tremaining: 92.4ms\n",
      "21:\tlearn: 0.2053547\ttotal: 25.5ms\tremaining: 90.4ms\n",
      "22:\tlearn: 0.1940625\ttotal: 26.4ms\tremaining: 88.4ms\n",
      "23:\tlearn: 0.1878110\ttotal: 27ms\tremaining: 85.4ms\n",
      "24:\tlearn: 0.1780354\ttotal: 27.6ms\tremaining: 83ms\n",
      "25:\tlearn: 0.1691095\ttotal: 28.2ms\tremaining: 80.3ms\n",
      "26:\tlearn: 0.1626937\ttotal: 29.1ms\tremaining: 78.6ms\n",
      "27:\tlearn: 0.1545721\ttotal: 30.2ms\tremaining: 77.6ms\n",
      "28:\tlearn: 0.1493414\ttotal: 31ms\tremaining: 75.9ms\n",
      "29:\tlearn: 0.1439116\ttotal: 31.8ms\tremaining: 74.3ms\n",
      "30:\tlearn: 0.1382508\ttotal: 32.4ms\tremaining: 72.2ms\n",
      "31:\tlearn: 0.1325987\ttotal: 33.2ms\tremaining: 70.5ms\n",
      "32:\tlearn: 0.1285720\ttotal: 34ms\tremaining: 69.1ms\n",
      "33:\tlearn: 0.1230360\ttotal: 34.8ms\tremaining: 67.5ms\n",
      "34:\tlearn: 0.1193629\ttotal: 35.6ms\tremaining: 66.1ms\n",
      "35:\tlearn: 0.1146339\ttotal: 36.3ms\tremaining: 64.6ms\n",
      "36:\tlearn: 0.1094729\ttotal: 37.1ms\tremaining: 63.2ms\n",
      "37:\tlearn: 0.1056727\ttotal: 37.8ms\tremaining: 61.7ms\n",
      "38:\tlearn: 0.1028650\ttotal: 38.6ms\tremaining: 60.3ms\n",
      "39:\tlearn: 0.0998269\ttotal: 39.3ms\tremaining: 58.9ms\n",
      "40:\tlearn: 0.0969805\ttotal: 40.1ms\tremaining: 57.8ms\n",
      "41:\tlearn: 0.0939673\ttotal: 40.9ms\tremaining: 56.4ms\n",
      "42:\tlearn: 0.0898606\ttotal: 41.5ms\tremaining: 55ms\n",
      "43:\tlearn: 0.0871762\ttotal: 42.3ms\tremaining: 53.8ms\n",
      "44:\tlearn: 0.0853154\ttotal: 43.1ms\tremaining: 52.6ms\n",
      "45:\tlearn: 0.0830705\ttotal: 43.8ms\tremaining: 51.4ms\n",
      "46:\tlearn: 0.0807763\ttotal: 44.7ms\tremaining: 50.4ms\n",
      "47:\tlearn: 0.0786132\ttotal: 45.4ms\tremaining: 49.2ms\n",
      "48:\tlearn: 0.0765113\ttotal: 46.1ms\tremaining: 48ms\n",
      "49:\tlearn: 0.0747012\ttotal: 46.9ms\tremaining: 46.9ms\n",
      "50:\tlearn: 0.0728028\ttotal: 47.9ms\tremaining: 46ms\n",
      "51:\tlearn: 0.0708340\ttotal: 48.9ms\tremaining: 45.2ms\n",
      "52:\tlearn: 0.0689617\ttotal: 49.8ms\tremaining: 44.2ms\n",
      "53:\tlearn: 0.0667731\ttotal: 50.6ms\tremaining: 43.1ms\n",
      "54:\tlearn: 0.0652038\ttotal: 51.5ms\tremaining: 42.2ms\n",
      "55:\tlearn: 0.0632232\ttotal: 52.4ms\tremaining: 41.1ms\n",
      "56:\tlearn: 0.0615428\ttotal: 53.9ms\tremaining: 40.6ms\n",
      "57:\tlearn: 0.0599641\ttotal: 54.7ms\tremaining: 39.6ms\n",
      "58:\tlearn: 0.0581723\ttotal: 55.4ms\tremaining: 38.5ms\n",
      "59:\tlearn: 0.0565808\ttotal: 56.2ms\tremaining: 37.4ms\n",
      "60:\tlearn: 0.0552709\ttotal: 56.9ms\tremaining: 36.4ms\n",
      "61:\tlearn: 0.0539313\ttotal: 57.7ms\tremaining: 35.4ms\n",
      "62:\tlearn: 0.0530169\ttotal: 58.4ms\tremaining: 34.3ms\n",
      "63:\tlearn: 0.0519182\ttotal: 59ms\tremaining: 33.2ms\n",
      "64:\tlearn: 0.0509428\ttotal: 59.9ms\tremaining: 32.3ms\n",
      "65:\tlearn: 0.0497314\ttotal: 60.7ms\tremaining: 31.3ms\n",
      "66:\tlearn: 0.0489236\ttotal: 61.3ms\tremaining: 30.2ms\n",
      "67:\tlearn: 0.0477973\ttotal: 62ms\tremaining: 29.2ms\n",
      "68:\tlearn: 0.0464654\ttotal: 62.7ms\tremaining: 28.2ms\n",
      "69:\tlearn: 0.0453462\ttotal: 63.4ms\tremaining: 27.2ms\n",
      "70:\tlearn: 0.0442542\ttotal: 64.1ms\tremaining: 26.2ms\n",
      "71:\tlearn: 0.0431962\ttotal: 64.7ms\tremaining: 25.2ms\n",
      "72:\tlearn: 0.0425099\ttotal: 65.4ms\tremaining: 24.2ms\n",
      "73:\tlearn: 0.0418933\ttotal: 66.1ms\tremaining: 23.2ms\n",
      "74:\tlearn: 0.0412187\ttotal: 66.8ms\tremaining: 22.3ms\n",
      "75:\tlearn: 0.0402741\ttotal: 67.4ms\tremaining: 21.3ms\n",
      "76:\tlearn: 0.0395916\ttotal: 68ms\tremaining: 20.3ms\n",
      "77:\tlearn: 0.0388844\ttotal: 68.6ms\tremaining: 19.3ms\n",
      "78:\tlearn: 0.0382083\ttotal: 69.2ms\tremaining: 18.4ms\n",
      "79:\tlearn: 0.0375428\ttotal: 69.8ms\tremaining: 17.4ms\n",
      "80:\tlearn: 0.0367988\ttotal: 70.5ms\tremaining: 16.5ms\n",
      "81:\tlearn: 0.0358785\ttotal: 71.1ms\tremaining: 15.6ms\n",
      "82:\tlearn: 0.0351277\ttotal: 71.8ms\tremaining: 14.7ms\n",
      "83:\tlearn: 0.0344429\ttotal: 72.4ms\tremaining: 13.8ms\n",
      "84:\tlearn: 0.0338459\ttotal: 73.1ms\tremaining: 12.9ms\n",
      "85:\tlearn: 0.0333656\ttotal: 73.9ms\tremaining: 12ms\n",
      "86:\tlearn: 0.0326905\ttotal: 74.5ms\tremaining: 11.1ms\n",
      "87:\tlearn: 0.0322153\ttotal: 75.2ms\tremaining: 10.3ms\n",
      "88:\tlearn: 0.0316731\ttotal: 75.8ms\tremaining: 9.36ms\n",
      "89:\tlearn: 0.0312643\ttotal: 76.4ms\tremaining: 8.48ms\n",
      "90:\tlearn: 0.0306622\ttotal: 76.9ms\tremaining: 7.61ms\n",
      "91:\tlearn: 0.0302142\ttotal: 77.6ms\tremaining: 6.75ms\n",
      "92:\tlearn: 0.0297389\ttotal: 78.3ms\tremaining: 5.89ms\n",
      "93:\tlearn: 0.0292045\ttotal: 78.9ms\tremaining: 5.04ms\n",
      "94:\tlearn: 0.0288106\ttotal: 79.6ms\tremaining: 4.19ms\n",
      "95:\tlearn: 0.0283491\ttotal: 80.2ms\tremaining: 3.34ms\n",
      "96:\tlearn: 0.0280041\ttotal: 80.8ms\tremaining: 2.5ms\n",
      "97:\tlearn: 0.0275969\ttotal: 81.4ms\tremaining: 1.66ms\n",
      "98:\tlearn: 0.0272802\ttotal: 82ms\tremaining: 828us\n",
      "99:\tlearn: 0.0269071\ttotal: 82.6ms\tremaining: 0us\n",
      "0:\tlearn: 0.9774634\ttotal: 1.26ms\tremaining: 125ms\n",
      "1:\tlearn: 0.8837130\ttotal: 2.67ms\tremaining: 131ms\n",
      "2:\tlearn: 0.8006016\ttotal: 3.98ms\tremaining: 129ms\n",
      "3:\tlearn: 0.7273705\ttotal: 5.11ms\tremaining: 123ms\n",
      "4:\tlearn: 0.6616135\ttotal: 6.13ms\tremaining: 116ms\n",
      "5:\tlearn: 0.6067632\ttotal: 7.21ms\tremaining: 113ms\n",
      "6:\tlearn: 0.5582865\ttotal: 8.53ms\tremaining: 113ms\n",
      "7:\tlearn: 0.5164546\ttotal: 9.85ms\tremaining: 113ms\n",
      "8:\tlearn: 0.4771826\ttotal: 11ms\tremaining: 111ms\n",
      "9:\tlearn: 0.4435987\ttotal: 12.7ms\tremaining: 114ms\n",
      "10:\tlearn: 0.4124402\ttotal: 14ms\tremaining: 113ms\n",
      "11:\tlearn: 0.3862891\ttotal: 15.2ms\tremaining: 112ms\n",
      "12:\tlearn: 0.3624465\ttotal: 16.4ms\tremaining: 110ms\n",
      "13:\tlearn: 0.3394638\ttotal: 17.4ms\tremaining: 107ms\n",
      "14:\tlearn: 0.3164623\ttotal: 18.4ms\tremaining: 104ms\n",
      "15:\tlearn: 0.2986806\ttotal: 19.3ms\tremaining: 101ms\n",
      "16:\tlearn: 0.2830617\ttotal: 20.3ms\tremaining: 99.2ms\n",
      "17:\tlearn: 0.2671743\ttotal: 21.3ms\tremaining: 97ms\n",
      "18:\tlearn: 0.2537053\ttotal: 22.3ms\tremaining: 94.9ms\n",
      "19:\tlearn: 0.2424665\ttotal: 23.4ms\tremaining: 93.7ms\n",
      "20:\tlearn: 0.2320760\ttotal: 24.4ms\tremaining: 91.7ms\n",
      "21:\tlearn: 0.2189963\ttotal: 25.3ms\tremaining: 89.5ms\n",
      "22:\tlearn: 0.2074273\ttotal: 26.2ms\tremaining: 87.7ms\n",
      "23:\tlearn: 0.1974916\ttotal: 27.3ms\tremaining: 86.5ms\n",
      "24:\tlearn: 0.1891814\ttotal: 28.5ms\tremaining: 85.5ms\n",
      "25:\tlearn: 0.1808325\ttotal: 29.4ms\tremaining: 83.8ms\n",
      "26:\tlearn: 0.1726887\ttotal: 30.4ms\tremaining: 82.2ms\n",
      "27:\tlearn: 0.1663402\ttotal: 31.4ms\tremaining: 80.6ms\n",
      "28:\tlearn: 0.1579216\ttotal: 32.3ms\tremaining: 79.2ms\n",
      "29:\tlearn: 0.1505515\ttotal: 33.3ms\tremaining: 77.8ms\n",
      "30:\tlearn: 0.1445967\ttotal: 34.2ms\tremaining: 76.2ms\n",
      "31:\tlearn: 0.1394076\ttotal: 35.4ms\tremaining: 75.2ms\n",
      "32:\tlearn: 0.1328516\ttotal: 36.5ms\tremaining: 74.1ms\n",
      "33:\tlearn: 0.1280514\ttotal: 37.6ms\tremaining: 73ms\n",
      "34:\tlearn: 0.1237178\ttotal: 38.6ms\tremaining: 71.7ms\n",
      "35:\tlearn: 0.1186936\ttotal: 39.5ms\tremaining: 70.1ms\n",
      "36:\tlearn: 0.1147735\ttotal: 40.2ms\tremaining: 68.5ms\n",
      "37:\tlearn: 0.1106278\ttotal: 41.1ms\tremaining: 67.1ms\n",
      "38:\tlearn: 0.1074415\ttotal: 42ms\tremaining: 65.7ms\n",
      "39:\tlearn: 0.1037704\ttotal: 42.8ms\tremaining: 64.2ms\n",
      "40:\tlearn: 0.1008700\ttotal: 43.2ms\tremaining: 62.2ms\n",
      "41:\tlearn: 0.0975588\ttotal: 44ms\tremaining: 60.8ms\n",
      "42:\tlearn: 0.0943958\ttotal: 44.9ms\tremaining: 59.5ms\n",
      "43:\tlearn: 0.0918444\ttotal: 45.9ms\tremaining: 58.4ms\n",
      "44:\tlearn: 0.0884760\ttotal: 46.8ms\tremaining: 57.2ms\n",
      "45:\tlearn: 0.0847699\ttotal: 47.6ms\tremaining: 55.9ms\n",
      "46:\tlearn: 0.0823142\ttotal: 48.5ms\tremaining: 54.6ms\n",
      "47:\tlearn: 0.0795014\ttotal: 49.3ms\tremaining: 53.4ms\n",
      "48:\tlearn: 0.0773169\ttotal: 50.2ms\tremaining: 52.3ms\n",
      "49:\tlearn: 0.0748414\ttotal: 51ms\tremaining: 51ms\n",
      "50:\tlearn: 0.0724460\ttotal: 51.9ms\tremaining: 49.9ms\n",
      "51:\tlearn: 0.0704300\ttotal: 52.7ms\tremaining: 48.7ms\n",
      "52:\tlearn: 0.0688136\ttotal: 53.5ms\tremaining: 47.5ms\n",
      "53:\tlearn: 0.0673299\ttotal: 54.3ms\tremaining: 46.3ms\n",
      "54:\tlearn: 0.0654950\ttotal: 55.3ms\tremaining: 45.2ms\n",
      "55:\tlearn: 0.0637919\ttotal: 56.3ms\tremaining: 44.2ms\n",
      "56:\tlearn: 0.0621312\ttotal: 56.9ms\tremaining: 42.9ms\n",
      "57:\tlearn: 0.0606361\ttotal: 57.8ms\tremaining: 41.8ms\n",
      "58:\tlearn: 0.0589670\ttotal: 58.6ms\tremaining: 40.7ms\n",
      "59:\tlearn: 0.0575501\ttotal: 59.7ms\tremaining: 39.8ms\n",
      "60:\tlearn: 0.0558869\ttotal: 60.7ms\tremaining: 38.8ms\n",
      "61:\tlearn: 0.0542316\ttotal: 61.8ms\tremaining: 37.9ms\n",
      "62:\tlearn: 0.0531384\ttotal: 62.9ms\tremaining: 36.9ms\n",
      "63:\tlearn: 0.0519472\ttotal: 63.9ms\tremaining: 36ms\n",
      "64:\tlearn: 0.0507477\ttotal: 65ms\tremaining: 35ms\n",
      "65:\tlearn: 0.0493451\ttotal: 65.4ms\tremaining: 33.7ms\n",
      "66:\tlearn: 0.0482952\ttotal: 66.3ms\tremaining: 32.6ms\n",
      "67:\tlearn: 0.0474708\ttotal: 67.1ms\tremaining: 31.6ms\n",
      "68:\tlearn: 0.0463083\ttotal: 68.2ms\tremaining: 30.6ms\n",
      "69:\tlearn: 0.0449767\ttotal: 69.2ms\tremaining: 29.7ms\n",
      "70:\tlearn: 0.0441092\ttotal: 70.1ms\tremaining: 28.6ms\n",
      "71:\tlearn: 0.0432024\ttotal: 70.8ms\tremaining: 27.5ms\n",
      "72:\tlearn: 0.0423087\ttotal: 71.6ms\tremaining: 26.5ms\n",
      "73:\tlearn: 0.0415555\ttotal: 72.3ms\tremaining: 25.4ms\n",
      "74:\tlearn: 0.0406448\ttotal: 73ms\tremaining: 24.3ms\n",
      "75:\tlearn: 0.0398417\ttotal: 73.9ms\tremaining: 23.3ms\n",
      "76:\tlearn: 0.0389946\ttotal: 74.7ms\tremaining: 22.3ms\n",
      "77:\tlearn: 0.0381767\ttotal: 75.5ms\tremaining: 21.3ms\n",
      "78:\tlearn: 0.0374057\ttotal: 76.3ms\tremaining: 20.3ms\n",
      "79:\tlearn: 0.0367360\ttotal: 77ms\tremaining: 19.3ms\n",
      "80:\tlearn: 0.0360360\ttotal: 77.9ms\tremaining: 18.3ms\n",
      "81:\tlearn: 0.0354539\ttotal: 78.6ms\tremaining: 17.3ms\n",
      "82:\tlearn: 0.0347557\ttotal: 79.3ms\tremaining: 16.2ms\n",
      "83:\tlearn: 0.0340954\ttotal: 80.1ms\tremaining: 15.3ms\n",
      "84:\tlearn: 0.0334014\ttotal: 81.2ms\tremaining: 14.3ms\n",
      "85:\tlearn: 0.0328912\ttotal: 82.2ms\tremaining: 13.4ms\n",
      "86:\tlearn: 0.0322561\ttotal: 83.2ms\tremaining: 12.4ms\n",
      "87:\tlearn: 0.0316911\ttotal: 84ms\tremaining: 11.5ms\n",
      "88:\tlearn: 0.0311513\ttotal: 84.8ms\tremaining: 10.5ms\n",
      "89:\tlearn: 0.0306558\ttotal: 85.7ms\tremaining: 9.52ms\n",
      "90:\tlearn: 0.0301222\ttotal: 86.4ms\tremaining: 8.54ms\n",
      "91:\tlearn: 0.0297150\ttotal: 87.1ms\tremaining: 7.57ms\n",
      "92:\tlearn: 0.0292251\ttotal: 87.9ms\tremaining: 6.61ms\n",
      "93:\tlearn: 0.0287529\ttotal: 88.6ms\tremaining: 5.66ms\n",
      "94:\tlearn: 0.0284144\ttotal: 89.3ms\tremaining: 4.7ms\n",
      "95:\tlearn: 0.0279712\ttotal: 90ms\tremaining: 3.75ms\n",
      "96:\tlearn: 0.0276372\ttotal: 90.9ms\tremaining: 2.81ms\n",
      "97:\tlearn: 0.0271599\ttotal: 92.5ms\tremaining: 1.89ms\n",
      "98:\tlearn: 0.0267420\ttotal: 93.3ms\tremaining: 942us\n",
      "99:\tlearn: 0.0263223\ttotal: 94ms\tremaining: 0us\n",
      "0:\tlearn: 0.9838338\ttotal: 1.27ms\tremaining: 126ms\n",
      "1:\tlearn: 0.8822870\ttotal: 2.91ms\tremaining: 143ms\n",
      "2:\tlearn: 0.7940753\ttotal: 3.69ms\tremaining: 119ms\n",
      "3:\tlearn: 0.7126448\ttotal: 4.5ms\tremaining: 108ms\n",
      "4:\tlearn: 0.6549573\ttotal: 5.83ms\tremaining: 111ms\n",
      "5:\tlearn: 0.5926869\ttotal: 7.15ms\tremaining: 112ms\n",
      "6:\tlearn: 0.5414832\ttotal: 8.61ms\tremaining: 114ms\n",
      "7:\tlearn: 0.4970902\ttotal: 10.1ms\tremaining: 116ms\n",
      "8:\tlearn: 0.4611538\ttotal: 11.7ms\tremaining: 118ms\n",
      "9:\tlearn: 0.4284653\ttotal: 13.1ms\tremaining: 118ms\n",
      "10:\tlearn: 0.3971098\ttotal: 14.2ms\tremaining: 115ms\n",
      "11:\tlearn: 0.3718981\ttotal: 15.4ms\tremaining: 113ms\n",
      "12:\tlearn: 0.3474071\ttotal: 16.7ms\tremaining: 112ms\n",
      "13:\tlearn: 0.3265971\ttotal: 18.1ms\tremaining: 111ms\n",
      "14:\tlearn: 0.3025381\ttotal: 19.5ms\tremaining: 110ms\n",
      "15:\tlearn: 0.2847464\ttotal: 20.8ms\tremaining: 109ms\n",
      "16:\tlearn: 0.2698415\ttotal: 22.1ms\tremaining: 108ms\n",
      "17:\tlearn: 0.2549864\ttotal: 22.7ms\tremaining: 103ms\n",
      "18:\tlearn: 0.2428546\ttotal: 23.8ms\tremaining: 102ms\n",
      "19:\tlearn: 0.2304568\ttotal: 24.9ms\tremaining: 99.8ms\n",
      "20:\tlearn: 0.2189009\ttotal: 26.1ms\tremaining: 98.3ms\n",
      "21:\tlearn: 0.2092618\ttotal: 27.3ms\tremaining: 96.6ms\n",
      "22:\tlearn: 0.1997479\ttotal: 28.5ms\tremaining: 95.3ms\n",
      "23:\tlearn: 0.1884690\ttotal: 29.3ms\tremaining: 92.7ms\n",
      "24:\tlearn: 0.1794992\ttotal: 30.3ms\tremaining: 91ms\n",
      "25:\tlearn: 0.1711057\ttotal: 31.4ms\tremaining: 89.4ms\n",
      "26:\tlearn: 0.1644384\ttotal: 32.4ms\tremaining: 87.6ms\n",
      "27:\tlearn: 0.1582695\ttotal: 33.5ms\tremaining: 86.1ms\n",
      "28:\tlearn: 0.1511962\ttotal: 34.5ms\tremaining: 84.5ms\n",
      "29:\tlearn: 0.1447028\ttotal: 35.8ms\tremaining: 83.4ms\n",
      "30:\tlearn: 0.1389118\ttotal: 37.1ms\tremaining: 82.6ms\n",
      "31:\tlearn: 0.1334825\ttotal: 37.9ms\tremaining: 80.6ms\n",
      "32:\tlearn: 0.1283928\ttotal: 39.1ms\tremaining: 79.4ms\n",
      "33:\tlearn: 0.1235856\ttotal: 40.3ms\tremaining: 78.2ms\n",
      "34:\tlearn: 0.1180339\ttotal: 41.4ms\tremaining: 77ms\n",
      "35:\tlearn: 0.1141932\ttotal: 42.6ms\tremaining: 75.7ms\n",
      "36:\tlearn: 0.1107089\ttotal: 43.7ms\tremaining: 74.4ms\n",
      "37:\tlearn: 0.1076034\ttotal: 44.7ms\tremaining: 72.9ms\n",
      "38:\tlearn: 0.1037160\ttotal: 45.6ms\tremaining: 71.4ms\n",
      "39:\tlearn: 0.0999564\ttotal: 46.6ms\tremaining: 70ms\n",
      "40:\tlearn: 0.0963829\ttotal: 47.6ms\tremaining: 68.5ms\n",
      "41:\tlearn: 0.0954076\ttotal: 48ms\tremaining: 66.3ms\n",
      "42:\tlearn: 0.0925346\ttotal: 48.9ms\tremaining: 64.9ms\n",
      "43:\tlearn: 0.0897160\ttotal: 49.9ms\tremaining: 63.5ms\n",
      "44:\tlearn: 0.0874719\ttotal: 51ms\tremaining: 62.3ms\n",
      "45:\tlearn: 0.0840241\ttotal: 51.6ms\tremaining: 60.6ms\n",
      "46:\tlearn: 0.0816898\ttotal: 52.6ms\tremaining: 59.3ms\n",
      "47:\tlearn: 0.0786714\ttotal: 53.5ms\tremaining: 58ms\n",
      "48:\tlearn: 0.0764372\ttotal: 54.5ms\tremaining: 56.7ms\n",
      "49:\tlearn: 0.0737888\ttotal: 55.7ms\tremaining: 55.7ms\n",
      "50:\tlearn: 0.0726082\ttotal: 56.9ms\tremaining: 54.7ms\n",
      "51:\tlearn: 0.0706356\ttotal: 57.9ms\tremaining: 53.5ms\n",
      "52:\tlearn: 0.0687126\ttotal: 58.9ms\tremaining: 52.2ms\n",
      "53:\tlearn: 0.0672109\ttotal: 60.1ms\tremaining: 51.2ms\n",
      "54:\tlearn: 0.0651909\ttotal: 61.3ms\tremaining: 50.1ms\n",
      "55:\tlearn: 0.0645274\ttotal: 62.3ms\tremaining: 49ms\n",
      "56:\tlearn: 0.0628959\ttotal: 63.8ms\tremaining: 48.1ms\n",
      "57:\tlearn: 0.0623256\ttotal: 64.6ms\tremaining: 46.7ms\n",
      "58:\tlearn: 0.0614382\ttotal: 65.3ms\tremaining: 45.4ms\n",
      "59:\tlearn: 0.0602164\ttotal: 66.4ms\tremaining: 44.3ms\n",
      "60:\tlearn: 0.0596381\ttotal: 67.7ms\tremaining: 43.3ms\n",
      "61:\tlearn: 0.0583011\ttotal: 68.9ms\tremaining: 42.2ms\n",
      "62:\tlearn: 0.0570549\ttotal: 70.1ms\tremaining: 41.1ms\n",
      "63:\tlearn: 0.0557242\ttotal: 71.1ms\tremaining: 40ms\n",
      "64:\tlearn: 0.0544177\ttotal: 72.1ms\tremaining: 38.8ms\n",
      "65:\tlearn: 0.0530603\ttotal: 73.2ms\tremaining: 37.7ms\n",
      "66:\tlearn: 0.0517333\ttotal: 74.1ms\tremaining: 36.5ms\n",
      "67:\tlearn: 0.0506770\ttotal: 75.1ms\tremaining: 35.3ms\n",
      "68:\tlearn: 0.0493059\ttotal: 76ms\tremaining: 34.1ms\n",
      "69:\tlearn: 0.0489621\ttotal: 76.8ms\tremaining: 32.9ms\n",
      "70:\tlearn: 0.0479741\ttotal: 77.6ms\tremaining: 31.7ms\n",
      "71:\tlearn: 0.0471727\ttotal: 78.4ms\tremaining: 30.5ms\n",
      "72:\tlearn: 0.0460616\ttotal: 79.1ms\tremaining: 29.3ms\n",
      "73:\tlearn: 0.0451228\ttotal: 80ms\tremaining: 28.1ms\n",
      "74:\tlearn: 0.0441745\ttotal: 80.7ms\tremaining: 26.9ms\n",
      "75:\tlearn: 0.0432612\ttotal: 81.4ms\tremaining: 25.7ms\n",
      "76:\tlearn: 0.0424448\ttotal: 82.2ms\tremaining: 24.5ms\n",
      "77:\tlearn: 0.0415357\ttotal: 83ms\tremaining: 23.4ms\n",
      "78:\tlearn: 0.0407802\ttotal: 83.8ms\tremaining: 22.3ms\n",
      "79:\tlearn: 0.0400732\ttotal: 84.5ms\tremaining: 21.1ms\n",
      "80:\tlearn: 0.0393917\ttotal: 85.3ms\tremaining: 20ms\n",
      "81:\tlearn: 0.0386159\ttotal: 86ms\tremaining: 18.9ms\n",
      "82:\tlearn: 0.0380056\ttotal: 86.8ms\tremaining: 17.8ms\n",
      "83:\tlearn: 0.0374274\ttotal: 87.6ms\tremaining: 16.7ms\n",
      "84:\tlearn: 0.0368509\ttotal: 88.2ms\tremaining: 15.6ms\n",
      "85:\tlearn: 0.0363361\ttotal: 89ms\tremaining: 14.5ms\n",
      "86:\tlearn: 0.0357783\ttotal: 89.7ms\tremaining: 13.4ms\n",
      "87:\tlearn: 0.0350711\ttotal: 90.5ms\tremaining: 12.3ms\n",
      "88:\tlearn: 0.0348171\ttotal: 91ms\tremaining: 11.2ms\n",
      "89:\tlearn: 0.0342729\ttotal: 91.7ms\tremaining: 10.2ms\n",
      "90:\tlearn: 0.0337674\ttotal: 92.4ms\tremaining: 9.14ms\n",
      "91:\tlearn: 0.0331244\ttotal: 93.1ms\tremaining: 8.1ms\n",
      "92:\tlearn: 0.0325352\ttotal: 93.8ms\tremaining: 7.06ms\n",
      "93:\tlearn: 0.0319837\ttotal: 94.5ms\tremaining: 6.03ms\n",
      "94:\tlearn: 0.0315356\ttotal: 95.2ms\tremaining: 5.01ms\n",
      "95:\tlearn: 0.0310480\ttotal: 95.9ms\tremaining: 4ms\n",
      "96:\tlearn: 0.0306035\ttotal: 96.6ms\tremaining: 2.99ms\n",
      "97:\tlearn: 0.0302192\ttotal: 97.2ms\tremaining: 1.98ms\n",
      "98:\tlearn: 0.0298123\ttotal: 97.6ms\tremaining: 985us\n",
      "99:\tlearn: 0.0293980\ttotal: 98.3ms\tremaining: 0us\n",
      "0:\tlearn: 0.9802605\ttotal: 1.24ms\tremaining: 122ms\n",
      "1:\tlearn: 0.8848801\ttotal: 2.73ms\tremaining: 134ms\n",
      "2:\tlearn: 0.7939699\ttotal: 3.46ms\tremaining: 112ms\n",
      "3:\tlearn: 0.7173107\ttotal: 4.64ms\tremaining: 111ms\n",
      "4:\tlearn: 0.6525783\ttotal: 5.79ms\tremaining: 110ms\n",
      "5:\tlearn: 0.5954232\ttotal: 6.89ms\tremaining: 108ms\n",
      "6:\tlearn: 0.5475409\ttotal: 8.06ms\tremaining: 107ms\n",
      "7:\tlearn: 0.5097047\ttotal: 9.05ms\tremaining: 104ms\n",
      "8:\tlearn: 0.4742693\ttotal: 10.1ms\tremaining: 102ms\n",
      "9:\tlearn: 0.4386181\ttotal: 11.3ms\tremaining: 101ms\n",
      "10:\tlearn: 0.4108689\ttotal: 12.5ms\tremaining: 101ms\n",
      "11:\tlearn: 0.3861533\ttotal: 13.7ms\tremaining: 100ms\n",
      "12:\tlearn: 0.3598841\ttotal: 14.9ms\tremaining: 99.5ms\n",
      "13:\tlearn: 0.3381099\ttotal: 15.9ms\tremaining: 97.5ms\n",
      "14:\tlearn: 0.3164765\ttotal: 16.9ms\tremaining: 95.7ms\n",
      "15:\tlearn: 0.2984563\ttotal: 18ms\tremaining: 94.6ms\n",
      "16:\tlearn: 0.2792274\ttotal: 19.1ms\tremaining: 93.5ms\n",
      "17:\tlearn: 0.2662162\ttotal: 20.2ms\tremaining: 91.9ms\n",
      "18:\tlearn: 0.2495170\ttotal: 21.2ms\tremaining: 90.2ms\n",
      "19:\tlearn: 0.2368829\ttotal: 22.3ms\tremaining: 89.1ms\n",
      "20:\tlearn: 0.2230133\ttotal: 23.3ms\tremaining: 87.7ms\n",
      "21:\tlearn: 0.2122630\ttotal: 24.4ms\tremaining: 86.4ms\n",
      "22:\tlearn: 0.2023791\ttotal: 25.5ms\tremaining: 85.2ms\n",
      "23:\tlearn: 0.1923354\ttotal: 26.5ms\tremaining: 83.8ms\n",
      "24:\tlearn: 0.1827374\ttotal: 27.4ms\tremaining: 82.2ms\n",
      "25:\tlearn: 0.1744903\ttotal: 28.4ms\tremaining: 80.8ms\n",
      "26:\tlearn: 0.1680341\ttotal: 29.3ms\tremaining: 79.3ms\n",
      "27:\tlearn: 0.1603743\ttotal: 30.4ms\tremaining: 78.3ms\n",
      "28:\tlearn: 0.1537826\ttotal: 32.4ms\tremaining: 79.3ms\n",
      "29:\tlearn: 0.1469019\ttotal: 33.5ms\tremaining: 78.3ms\n",
      "30:\tlearn: 0.1408981\ttotal: 34.6ms\tremaining: 77.1ms\n",
      "31:\tlearn: 0.1355597\ttotal: 35.5ms\tremaining: 75.5ms\n",
      "32:\tlearn: 0.1301404\ttotal: 36.5ms\tremaining: 74.1ms\n",
      "33:\tlearn: 0.1245551\ttotal: 37.3ms\tremaining: 72.5ms\n",
      "34:\tlearn: 0.1197676\ttotal: 38.2ms\tremaining: 70.9ms\n",
      "35:\tlearn: 0.1158267\ttotal: 39.1ms\tremaining: 69.4ms\n",
      "36:\tlearn: 0.1115688\ttotal: 40ms\tremaining: 68.1ms\n",
      "37:\tlearn: 0.1067352\ttotal: 40.9ms\tremaining: 66.7ms\n",
      "38:\tlearn: 0.1027453\ttotal: 41.9ms\tremaining: 65.5ms\n",
      "39:\tlearn: 0.0986659\ttotal: 42.9ms\tremaining: 64.4ms\n",
      "40:\tlearn: 0.0953609\ttotal: 43.5ms\tremaining: 62.6ms\n",
      "41:\tlearn: 0.0919401\ttotal: 44.4ms\tremaining: 61.3ms\n",
      "42:\tlearn: 0.0891051\ttotal: 45.3ms\tremaining: 60ms\n",
      "43:\tlearn: 0.0864185\ttotal: 46.2ms\tremaining: 58.8ms\n",
      "44:\tlearn: 0.0829951\ttotal: 47.1ms\tremaining: 57.6ms\n",
      "45:\tlearn: 0.0800063\ttotal: 48.2ms\tremaining: 56.5ms\n",
      "46:\tlearn: 0.0777944\ttotal: 49.2ms\tremaining: 55.5ms\n",
      "47:\tlearn: 0.0755068\ttotal: 50.1ms\tremaining: 54.3ms\n",
      "48:\tlearn: 0.0735075\ttotal: 51ms\tremaining: 53.1ms\n",
      "49:\tlearn: 0.0710304\ttotal: 51.9ms\tremaining: 51.9ms\n",
      "50:\tlearn: 0.0687439\ttotal: 52.6ms\tremaining: 50.5ms\n",
      "51:\tlearn: 0.0663910\ttotal: 53.2ms\tremaining: 49.1ms\n",
      "52:\tlearn: 0.0649097\ttotal: 54ms\tremaining: 47.9ms\n",
      "53:\tlearn: 0.0630295\ttotal: 54.8ms\tremaining: 46.7ms\n",
      "54:\tlearn: 0.0613444\ttotal: 55.7ms\tremaining: 45.6ms\n",
      "55:\tlearn: 0.0597607\ttotal: 56.5ms\tremaining: 44.4ms\n",
      "56:\tlearn: 0.0585184\ttotal: 57.4ms\tremaining: 43.3ms\n",
      "57:\tlearn: 0.0571888\ttotal: 58.2ms\tremaining: 42.2ms\n",
      "58:\tlearn: 0.0557193\ttotal: 59.1ms\tremaining: 41.1ms\n",
      "59:\tlearn: 0.0544705\ttotal: 60ms\tremaining: 40ms\n",
      "60:\tlearn: 0.0529673\ttotal: 60.8ms\tremaining: 38.9ms\n",
      "61:\tlearn: 0.0518933\ttotal: 61.6ms\tremaining: 37.8ms\n",
      "62:\tlearn: 0.0504902\ttotal: 62.2ms\tremaining: 36.5ms\n",
      "63:\tlearn: 0.0492981\ttotal: 63.1ms\tremaining: 35.5ms\n",
      "64:\tlearn: 0.0480975\ttotal: 64.2ms\tremaining: 34.6ms\n",
      "65:\tlearn: 0.0471845\ttotal: 65.2ms\tremaining: 33.6ms\n",
      "66:\tlearn: 0.0462157\ttotal: 66ms\tremaining: 32.5ms\n",
      "67:\tlearn: 0.0452823\ttotal: 67ms\tremaining: 31.5ms\n",
      "68:\tlearn: 0.0441800\ttotal: 67.7ms\tremaining: 30.4ms\n",
      "69:\tlearn: 0.0431540\ttotal: 68.6ms\tremaining: 29.4ms\n",
      "70:\tlearn: 0.0421109\ttotal: 69.5ms\tremaining: 28.4ms\n",
      "71:\tlearn: 0.0411946\ttotal: 70.3ms\tremaining: 27.3ms\n",
      "72:\tlearn: 0.0403934\ttotal: 71.2ms\tremaining: 26.3ms\n",
      "73:\tlearn: 0.0395462\ttotal: 72ms\tremaining: 25.3ms\n",
      "74:\tlearn: 0.0388264\ttotal: 72.8ms\tremaining: 24.3ms\n",
      "75:\tlearn: 0.0383015\ttotal: 73.6ms\tremaining: 23.2ms\n",
      "76:\tlearn: 0.0376224\ttotal: 74.3ms\tremaining: 22.2ms\n",
      "77:\tlearn: 0.0368396\ttotal: 75.1ms\tremaining: 21.2ms\n",
      "78:\tlearn: 0.0360968\ttotal: 75.9ms\tremaining: 20.2ms\n",
      "79:\tlearn: 0.0352932\ttotal: 77.5ms\tremaining: 19.4ms\n",
      "80:\tlearn: 0.0345032\ttotal: 78.4ms\tremaining: 18.4ms\n",
      "81:\tlearn: 0.0338650\ttotal: 79ms\tremaining: 17.4ms\n",
      "82:\tlearn: 0.0332438\ttotal: 79.8ms\tremaining: 16.4ms\n",
      "83:\tlearn: 0.0327107\ttotal: 80.7ms\tremaining: 15.4ms\n",
      "84:\tlearn: 0.0320408\ttotal: 81.5ms\tremaining: 14.4ms\n",
      "85:\tlearn: 0.0315518\ttotal: 82.3ms\tremaining: 13.4ms\n",
      "86:\tlearn: 0.0309987\ttotal: 83.1ms\tremaining: 12.4ms\n",
      "87:\tlearn: 0.0305056\ttotal: 83.8ms\tremaining: 11.4ms\n",
      "88:\tlearn: 0.0300386\ttotal: 84.6ms\tremaining: 10.5ms\n",
      "89:\tlearn: 0.0295569\ttotal: 85.5ms\tremaining: 9.49ms\n",
      "90:\tlearn: 0.0291072\ttotal: 86.3ms\tremaining: 8.53ms\n",
      "91:\tlearn: 0.0287268\ttotal: 87.1ms\tremaining: 7.58ms\n",
      "92:\tlearn: 0.0282308\ttotal: 87.9ms\tremaining: 6.62ms\n",
      "93:\tlearn: 0.0276486\ttotal: 88.7ms\tremaining: 5.66ms\n",
      "94:\tlearn: 0.0272248\ttotal: 89.4ms\tremaining: 4.71ms\n",
      "95:\tlearn: 0.0268967\ttotal: 90.3ms\tremaining: 3.76ms\n",
      "96:\tlearn: 0.0264565\ttotal: 91.1ms\tremaining: 2.82ms\n",
      "97:\tlearn: 0.0261531\ttotal: 91.9ms\tremaining: 1.88ms\n",
      "98:\tlearn: 0.0257289\ttotal: 92.7ms\tremaining: 936us\n",
      "99:\tlearn: 0.0253478\ttotal: 93.4ms\tremaining: 0us\n",
      "0:\tlearn: 0.9765021\ttotal: 1.16ms\tremaining: 115ms\n",
      "1:\tlearn: 0.8756253\ttotal: 3.09ms\tremaining: 151ms\n",
      "2:\tlearn: 0.7970387\ttotal: 9.99ms\tremaining: 323ms\n",
      "3:\tlearn: 0.7213265\ttotal: 11.4ms\tremaining: 274ms\n",
      "4:\tlearn: 0.6568614\ttotal: 12.5ms\tremaining: 238ms\n",
      "5:\tlearn: 0.6035829\ttotal: 13.9ms\tremaining: 218ms\n",
      "6:\tlearn: 0.5569064\ttotal: 15.1ms\tremaining: 200ms\n",
      "7:\tlearn: 0.5163493\ttotal: 16.2ms\tremaining: 186ms\n",
      "8:\tlearn: 0.4748696\ttotal: 17.1ms\tremaining: 173ms\n",
      "9:\tlearn: 0.4394957\ttotal: 18ms\tremaining: 162ms\n",
      "10:\tlearn: 0.4082545\ttotal: 19ms\tremaining: 153ms\n",
      "11:\tlearn: 0.3807652\ttotal: 20.2ms\tremaining: 148ms\n",
      "12:\tlearn: 0.3560968\ttotal: 21.3ms\tremaining: 143ms\n",
      "13:\tlearn: 0.3348663\ttotal: 22.3ms\tremaining: 137ms\n",
      "14:\tlearn: 0.3135797\ttotal: 23.2ms\tremaining: 132ms\n",
      "15:\tlearn: 0.2954293\ttotal: 24.1ms\tremaining: 126ms\n",
      "16:\tlearn: 0.2794337\ttotal: 25ms\tremaining: 122ms\n",
      "17:\tlearn: 0.2638442\ttotal: 25.8ms\tremaining: 118ms\n",
      "18:\tlearn: 0.2494471\ttotal: 26.7ms\tremaining: 114ms\n",
      "19:\tlearn: 0.2375660\ttotal: 27.5ms\tremaining: 110ms\n",
      "20:\tlearn: 0.2248028\ttotal: 28.3ms\tremaining: 107ms\n",
      "21:\tlearn: 0.2136366\ttotal: 29.2ms\tremaining: 103ms\n",
      "22:\tlearn: 0.2017887\ttotal: 30ms\tremaining: 100ms\n",
      "23:\tlearn: 0.1936202\ttotal: 31.5ms\tremaining: 99.7ms\n",
      "24:\tlearn: 0.1838557\ttotal: 32.4ms\tremaining: 97.3ms\n",
      "25:\tlearn: 0.1762732\ttotal: 33.3ms\tremaining: 94.7ms\n",
      "26:\tlearn: 0.1691196\ttotal: 34.2ms\tremaining: 92.3ms\n",
      "27:\tlearn: 0.1622514\ttotal: 35ms\tremaining: 90ms\n",
      "28:\tlearn: 0.1545962\ttotal: 36ms\tremaining: 88ms\n",
      "29:\tlearn: 0.1489286\ttotal: 36.8ms\tremaining: 85.9ms\n",
      "30:\tlearn: 0.1417020\ttotal: 37.6ms\tremaining: 83.7ms\n",
      "31:\tlearn: 0.1364662\ttotal: 38.8ms\tremaining: 82.5ms\n",
      "32:\tlearn: 0.1319730\ttotal: 39.9ms\tremaining: 81ms\n",
      "33:\tlearn: 0.1267868\ttotal: 40.8ms\tremaining: 79.2ms\n",
      "34:\tlearn: 0.1226359\ttotal: 41.8ms\tremaining: 77.7ms\n",
      "35:\tlearn: 0.1183709\ttotal: 42.5ms\tremaining: 75.6ms\n",
      "36:\tlearn: 0.1154904\ttotal: 43.4ms\tremaining: 73.9ms\n",
      "37:\tlearn: 0.1117346\ttotal: 44.5ms\tremaining: 72.6ms\n",
      "38:\tlearn: 0.1078852\ttotal: 45.5ms\tremaining: 71.1ms\n",
      "39:\tlearn: 0.1042394\ttotal: 46.3ms\tremaining: 69.5ms\n",
      "40:\tlearn: 0.1011149\ttotal: 47.2ms\tremaining: 67.9ms\n",
      "41:\tlearn: 0.0978632\ttotal: 48.1ms\tremaining: 66.4ms\n",
      "42:\tlearn: 0.0946660\ttotal: 48.9ms\tremaining: 64.8ms\n",
      "43:\tlearn: 0.0919581\ttotal: 49.8ms\tremaining: 63.4ms\n",
      "44:\tlearn: 0.0891048\ttotal: 50.7ms\tremaining: 62ms\n",
      "45:\tlearn: 0.0856745\ttotal: 51.6ms\tremaining: 60.6ms\n",
      "46:\tlearn: 0.0825291\ttotal: 52.6ms\tremaining: 59.3ms\n",
      "47:\tlearn: 0.0801332\ttotal: 53.5ms\tremaining: 57.9ms\n",
      "48:\tlearn: 0.0780203\ttotal: 54.2ms\tremaining: 56.4ms\n",
      "49:\tlearn: 0.0764840\ttotal: 54.9ms\tremaining: 54.9ms\n",
      "50:\tlearn: 0.0739952\ttotal: 55.6ms\tremaining: 53.4ms\n",
      "51:\tlearn: 0.0720814\ttotal: 56.4ms\tremaining: 52ms\n",
      "52:\tlearn: 0.0704357\ttotal: 57.1ms\tremaining: 50.6ms\n",
      "53:\tlearn: 0.0690679\ttotal: 57.8ms\tremaining: 49.2ms\n",
      "54:\tlearn: 0.0674258\ttotal: 58.5ms\tremaining: 47.9ms\n",
      "55:\tlearn: 0.0661019\ttotal: 59.4ms\tremaining: 46.6ms\n",
      "56:\tlearn: 0.0644244\ttotal: 60.1ms\tremaining: 45.4ms\n",
      "57:\tlearn: 0.0632688\ttotal: 60.8ms\tremaining: 44.1ms\n",
      "58:\tlearn: 0.0616830\ttotal: 61.6ms\tremaining: 42.8ms\n",
      "59:\tlearn: 0.0602591\ttotal: 62.3ms\tremaining: 41.5ms\n",
      "60:\tlearn: 0.0584895\ttotal: 63.1ms\tremaining: 40.3ms\n",
      "61:\tlearn: 0.0571111\ttotal: 64.1ms\tremaining: 39.3ms\n",
      "62:\tlearn: 0.0560758\ttotal: 65ms\tremaining: 38.2ms\n",
      "63:\tlearn: 0.0548416\ttotal: 65.8ms\tremaining: 37ms\n",
      "64:\tlearn: 0.0537095\ttotal: 66.6ms\tremaining: 35.8ms\n",
      "65:\tlearn: 0.0524313\ttotal: 67.4ms\tremaining: 34.7ms\n",
      "66:\tlearn: 0.0513114\ttotal: 68.2ms\tremaining: 33.6ms\n",
      "67:\tlearn: 0.0508206\ttotal: 68.7ms\tremaining: 32.3ms\n",
      "68:\tlearn: 0.0502981\ttotal: 69.7ms\tremaining: 31.3ms\n",
      "69:\tlearn: 0.0489944\ttotal: 70.5ms\tremaining: 30.2ms\n",
      "70:\tlearn: 0.0480419\ttotal: 71.2ms\tremaining: 29.1ms\n",
      "71:\tlearn: 0.0471925\ttotal: 72.1ms\tremaining: 28ms\n",
      "72:\tlearn: 0.0464420\ttotal: 73ms\tremaining: 27ms\n",
      "73:\tlearn: 0.0456376\ttotal: 73.8ms\tremaining: 25.9ms\n",
      "74:\tlearn: 0.0447408\ttotal: 74.6ms\tremaining: 24.9ms\n",
      "75:\tlearn: 0.0436899\ttotal: 75.3ms\tremaining: 23.8ms\n",
      "76:\tlearn: 0.0429111\ttotal: 76.1ms\tremaining: 22.7ms\n",
      "77:\tlearn: 0.0420582\ttotal: 76.8ms\tremaining: 21.7ms\n",
      "78:\tlearn: 0.0413441\ttotal: 77.7ms\tremaining: 20.7ms\n",
      "79:\tlearn: 0.0405545\ttotal: 78.6ms\tremaining: 19.6ms\n",
      "80:\tlearn: 0.0395691\ttotal: 79.4ms\tremaining: 18.6ms\n",
      "81:\tlearn: 0.0388639\ttotal: 80.3ms\tremaining: 17.6ms\n",
      "82:\tlearn: 0.0382766\ttotal: 81.1ms\tremaining: 16.6ms\n",
      "83:\tlearn: 0.0375734\ttotal: 81.9ms\tremaining: 15.6ms\n",
      "84:\tlearn: 0.0368438\ttotal: 82.6ms\tremaining: 14.6ms\n",
      "85:\tlearn: 0.0363279\ttotal: 83.3ms\tremaining: 13.6ms\n",
      "86:\tlearn: 0.0356425\ttotal: 84ms\tremaining: 12.5ms\n",
      "87:\tlearn: 0.0350696\ttotal: 84.7ms\tremaining: 11.6ms\n",
      "88:\tlearn: 0.0344648\ttotal: 85.4ms\tremaining: 10.5ms\n",
      "89:\tlearn: 0.0339049\ttotal: 86.6ms\tremaining: 9.62ms\n",
      "90:\tlearn: 0.0333421\ttotal: 87.5ms\tremaining: 8.66ms\n",
      "91:\tlearn: 0.0328959\ttotal: 88.3ms\tremaining: 7.68ms\n",
      "92:\tlearn: 0.0326414\ttotal: 89.1ms\tremaining: 6.7ms\n",
      "93:\tlearn: 0.0320741\ttotal: 89.8ms\tremaining: 5.73ms\n",
      "94:\tlearn: 0.0317127\ttotal: 90.6ms\tremaining: 4.76ms\n",
      "95:\tlearn: 0.0312847\ttotal: 91.2ms\tremaining: 3.8ms\n",
      "96:\tlearn: 0.0308849\ttotal: 91.7ms\tremaining: 2.84ms\n",
      "97:\tlearn: 0.0305419\ttotal: 92.4ms\tremaining: 1.89ms\n",
      "98:\tlearn: 0.0300625\ttotal: 93ms\tremaining: 939us\n",
      "99:\tlearn: 0.0295228\ttotal: 93.6ms\tremaining: 0us\n",
      "0:\tlearn: 0.9699968\ttotal: 1.83ms\tremaining: 182ms\n",
      "1:\tlearn: 0.8563335\ttotal: 3ms\tremaining: 147ms\n",
      "2:\tlearn: 0.7826969\ttotal: 4.26ms\tremaining: 138ms\n",
      "3:\tlearn: 0.7067632\ttotal: 5.58ms\tremaining: 134ms\n",
      "4:\tlearn: 0.6340604\ttotal: 6.61ms\tremaining: 125ms\n",
      "5:\tlearn: 0.5727775\ttotal: 7.84ms\tremaining: 123ms\n",
      "6:\tlearn: 0.5264836\ttotal: 8.79ms\tremaining: 117ms\n",
      "7:\tlearn: 0.4886505\ttotal: 10ms\tremaining: 115ms\n",
      "8:\tlearn: 0.4525170\ttotal: 11.3ms\tremaining: 114ms\n",
      "9:\tlearn: 0.4163326\ttotal: 12.6ms\tremaining: 113ms\n",
      "10:\tlearn: 0.3858267\ttotal: 14ms\tremaining: 114ms\n",
      "11:\tlearn: 0.3628084\ttotal: 15.3ms\tremaining: 112ms\n",
      "12:\tlearn: 0.3363728\ttotal: 16.6ms\tremaining: 111ms\n",
      "13:\tlearn: 0.3157877\ttotal: 17.8ms\tremaining: 109ms\n",
      "14:\tlearn: 0.2972902\ttotal: 19ms\tremaining: 108ms\n",
      "15:\tlearn: 0.2806537\ttotal: 20.3ms\tremaining: 106ms\n",
      "16:\tlearn: 0.2616100\ttotal: 21.5ms\tremaining: 105ms\n",
      "17:\tlearn: 0.2461832\ttotal: 22.7ms\tremaining: 103ms\n",
      "18:\tlearn: 0.2317244\ttotal: 24ms\tremaining: 103ms\n",
      "19:\tlearn: 0.2177199\ttotal: 25.3ms\tremaining: 101ms\n",
      "20:\tlearn: 0.2049528\ttotal: 26.4ms\tremaining: 99.4ms\n",
      "21:\tlearn: 0.1938791\ttotal: 27.7ms\tremaining: 98.2ms\n",
      "22:\tlearn: 0.1850573\ttotal: 29.1ms\tremaining: 97.4ms\n",
      "23:\tlearn: 0.1750273\ttotal: 30.4ms\tremaining: 96.3ms\n",
      "24:\tlearn: 0.1660575\ttotal: 31.7ms\tremaining: 95.2ms\n",
      "25:\tlearn: 0.1575991\ttotal: 33.1ms\tremaining: 94.2ms\n",
      "26:\tlearn: 0.1511252\ttotal: 34.2ms\tremaining: 92.4ms\n",
      "27:\tlearn: 0.1444636\ttotal: 35.4ms\tremaining: 91.1ms\n",
      "28:\tlearn: 0.1379228\ttotal: 36.2ms\tremaining: 88.5ms\n",
      "29:\tlearn: 0.1314975\ttotal: 37.3ms\tremaining: 87ms\n",
      "30:\tlearn: 0.1262929\ttotal: 38.3ms\tremaining: 85.3ms\n",
      "31:\tlearn: 0.1212339\ttotal: 39.4ms\tremaining: 83.8ms\n",
      "32:\tlearn: 0.1162073\ttotal: 40.5ms\tremaining: 82.2ms\n",
      "33:\tlearn: 0.1110441\ttotal: 41.5ms\tremaining: 80.6ms\n",
      "34:\tlearn: 0.1065761\ttotal: 42.4ms\tremaining: 78.7ms\n",
      "35:\tlearn: 0.1026064\ttotal: 43.2ms\tremaining: 76.8ms\n",
      "36:\tlearn: 0.0987313\ttotal: 44.2ms\tremaining: 75.3ms\n",
      "37:\tlearn: 0.0951692\ttotal: 45.2ms\tremaining: 73.7ms\n",
      "38:\tlearn: 0.0918070\ttotal: 46ms\tremaining: 72ms\n",
      "39:\tlearn: 0.0881494\ttotal: 47ms\tremaining: 70.5ms\n",
      "40:\tlearn: 0.0852959\ttotal: 47.9ms\tremaining: 68.9ms\n",
      "41:\tlearn: 0.0826698\ttotal: 48.8ms\tremaining: 67.4ms\n",
      "42:\tlearn: 0.0800733\ttotal: 49.8ms\tremaining: 66ms\n",
      "43:\tlearn: 0.0780253\ttotal: 50.6ms\tremaining: 64.5ms\n",
      "44:\tlearn: 0.0759622\ttotal: 51.5ms\tremaining: 63ms\n",
      "45:\tlearn: 0.0737626\ttotal: 52.3ms\tremaining: 61.4ms\n",
      "46:\tlearn: 0.0715721\ttotal: 52.9ms\tremaining: 59.6ms\n",
      "47:\tlearn: 0.0693454\ttotal: 54.5ms\tremaining: 59.1ms\n",
      "48:\tlearn: 0.0675104\ttotal: 55.6ms\tremaining: 57.9ms\n",
      "49:\tlearn: 0.0654927\ttotal: 56.4ms\tremaining: 56.4ms\n",
      "50:\tlearn: 0.0639499\ttotal: 57.3ms\tremaining: 55.1ms\n",
      "51:\tlearn: 0.0623373\ttotal: 58.3ms\tremaining: 53.8ms\n",
      "52:\tlearn: 0.0606871\ttotal: 59ms\tremaining: 52.3ms\n",
      "53:\tlearn: 0.0593723\ttotal: 60ms\tremaining: 51.1ms\n",
      "54:\tlearn: 0.0576296\ttotal: 61ms\tremaining: 49.9ms\n",
      "55:\tlearn: 0.0558253\ttotal: 62ms\tremaining: 48.7ms\n",
      "56:\tlearn: 0.0540826\ttotal: 63ms\tremaining: 47.5ms\n",
      "57:\tlearn: 0.0528423\ttotal: 64.1ms\tremaining: 46.4ms\n",
      "58:\tlearn: 0.0517677\ttotal: 65ms\tremaining: 45.2ms\n",
      "59:\tlearn: 0.0502063\ttotal: 66ms\tremaining: 44ms\n",
      "60:\tlearn: 0.0490790\ttotal: 67.2ms\tremaining: 42.9ms\n",
      "61:\tlearn: 0.0477336\ttotal: 68.2ms\tremaining: 41.8ms\n",
      "62:\tlearn: 0.0467304\ttotal: 69.3ms\tremaining: 40.7ms\n",
      "63:\tlearn: 0.0457733\ttotal: 70.1ms\tremaining: 39.4ms\n",
      "64:\tlearn: 0.0447246\ttotal: 71ms\tremaining: 38.3ms\n",
      "65:\tlearn: 0.0436893\ttotal: 72ms\tremaining: 37.1ms\n",
      "66:\tlearn: 0.0427062\ttotal: 72.8ms\tremaining: 35.9ms\n",
      "67:\tlearn: 0.0417712\ttotal: 73.6ms\tremaining: 34.6ms\n",
      "68:\tlearn: 0.0408905\ttotal: 74.3ms\tremaining: 33.4ms\n",
      "69:\tlearn: 0.0398336\ttotal: 74.7ms\tremaining: 32ms\n",
      "70:\tlearn: 0.0389808\ttotal: 75.4ms\tremaining: 30.8ms\n",
      "71:\tlearn: 0.0380119\ttotal: 76.2ms\tremaining: 29.6ms\n",
      "72:\tlearn: 0.0371847\ttotal: 77ms\tremaining: 28.5ms\n",
      "73:\tlearn: 0.0363869\ttotal: 77.8ms\tremaining: 27.3ms\n",
      "74:\tlearn: 0.0355559\ttotal: 78.6ms\tremaining: 26.2ms\n",
      "75:\tlearn: 0.0348831\ttotal: 79.1ms\tremaining: 25ms\n",
      "76:\tlearn: 0.0342071\ttotal: 80ms\tremaining: 23.9ms\n",
      "77:\tlearn: 0.0335253\ttotal: 80.8ms\tremaining: 22.8ms\n",
      "78:\tlearn: 0.0329172\ttotal: 81.5ms\tremaining: 21.7ms\n",
      "79:\tlearn: 0.0322944\ttotal: 82.2ms\tremaining: 20.5ms\n",
      "80:\tlearn: 0.0317213\ttotal: 83.2ms\tremaining: 19.5ms\n",
      "81:\tlearn: 0.0312121\ttotal: 83.9ms\tremaining: 18.4ms\n",
      "82:\tlearn: 0.0305626\ttotal: 84.8ms\tremaining: 17.4ms\n",
      "83:\tlearn: 0.0301265\ttotal: 85.6ms\tremaining: 16.3ms\n",
      "84:\tlearn: 0.0296273\ttotal: 86.6ms\tremaining: 15.3ms\n",
      "85:\tlearn: 0.0291537\ttotal: 87.3ms\tremaining: 14.2ms\n",
      "86:\tlearn: 0.0286039\ttotal: 88.1ms\tremaining: 13.2ms\n",
      "87:\tlearn: 0.0280922\ttotal: 88.9ms\tremaining: 12.1ms\n",
      "88:\tlearn: 0.0275677\ttotal: 89.4ms\tremaining: 11ms\n",
      "89:\tlearn: 0.0271217\ttotal: 90.1ms\tremaining: 10ms\n",
      "90:\tlearn: 0.0267128\ttotal: 90.8ms\tremaining: 8.98ms\n",
      "91:\tlearn: 0.0263922\ttotal: 91.6ms\tremaining: 7.97ms\n",
      "92:\tlearn: 0.0259871\ttotal: 92.4ms\tremaining: 6.96ms\n",
      "93:\tlearn: 0.0256218\ttotal: 93.1ms\tremaining: 5.94ms\n",
      "94:\tlearn: 0.0252923\ttotal: 93.8ms\tremaining: 4.94ms\n",
      "95:\tlearn: 0.0249506\ttotal: 94.5ms\tremaining: 3.94ms\n",
      "96:\tlearn: 0.0245916\ttotal: 95.2ms\tremaining: 2.94ms\n",
      "97:\tlearn: 0.0242821\ttotal: 95.8ms\tremaining: 1.96ms\n",
      "98:\tlearn: 0.0238932\ttotal: 96.2ms\tremaining: 971us\n",
      "99:\tlearn: 0.0235904\ttotal: 97ms\tremaining: 0us\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0   1.0   2.0\n",
      "0.0  20.0   0.0   0.0\n",
      "1.0   0.0  19.0   0.0\n",
      "2.0   0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  1.0\n",
      "Precision total:  1.0\n",
      "Recall total:  1.0\n",
      "F1 total:  1.0\n",
      "BACC total:  1.0\n",
      "MCC total:  1.0\n"
     ]
    }
   ],
   "source": [
    "import catboost\n",
    "start = time.time()\n",
    "\n",
    "bag_cat = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "\n",
    "base_classifier = bag_cat\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_cat'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Baggin LGBM\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0   1.0   2.0\n",
      "0.0  20.0   0.0   0.0\n",
      "1.0   1.0  18.0   0.0\n",
      "2.0   0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9818181818181818\n",
      "Precision total:  0.9841269841269842\n",
      "Recall total:  0.9824561403508771\n",
      "F1 total:  0.9828609096901779\n",
      "BACC total:  0.9824561403508771\n",
      "MCC total:  0.973081241361401\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "from lightgbm import LGBMClassifier\n",
    "lgbm = LGBMClassifier()\n",
    "\n",
    "\n",
    "base_classifier = lgbm\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_lgbm'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# import xgboost as xgb\n",
    "\n",
    "# # Create a DMatrix for XGBoost\n",
    "# dtrain = xgb.DMatrix(X_train_01, label=y_train_01)\n",
    "# dtest = xgb.DMatrix(X_test_01, label=y_test_01)\n",
    "\n",
    "# # Set XGBoost parameters\n",
    "# params = {\n",
    "#     'objective': 'multi:softmax',  # for multi-class classification\n",
    "#     'num_class': 5,  # specify the number of classes\n",
    "#     'max_depth': 3,\n",
    "#     'learning_rate': 0.1,\n",
    "#     'eval_metric': 'mlogloss'  # metric for multi-class classification\n",
    "# }\n",
    "\n",
    "# # Train the XGBoost model\n",
    "# num_round = 100\n",
    "# xgb_01 = xgb.train(params, dtrain, num_round)\n",
    "\n",
    "# base_classifier = xgb\n",
    "\n",
    "# # Define the BaggingClassifier\n",
    "# bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# # Train the BaggingClassifier\n",
    "# bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# # Make predictions on the test set\n",
    "# y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "# name = 'bag_xgb'\n",
    "\n",
    "# pred_label = y_pred\n",
    "\n",
    "\n",
    "# metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "# Acc = metrics[0]\n",
    "# Precision = metrics[1]\n",
    "# Recall = metrics[2]\n",
    "# F1 = metrics[3]\n",
    "# BACC = metrics[4]\n",
    "# MCC = metrics[5]    \n",
    "\n",
    "\n",
    "# globals()[f\"{name}_acc_01\"] = Acc\n",
    "# globals()[f\"{name}_pre_01\"] = Precision\n",
    "# globals()[f\"{name}_rec_01\"] = Recall\n",
    "# globals()[f\"{name}_f1_01\"] = F1\n",
    "# globals()[f\"{name}_bacc_01\"] = BACC\n",
    "# globals()[f\"{name}_mcc_01\"] = MCC\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0   1.0   2.0\n",
      "0.0  20.0   0.0   0.0\n",
      "1.0   0.0  19.0   0.0\n",
      "2.0   0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  1.0\n",
      "Precision total:  1.0\n",
      "Recall total:  1.0\n",
      "F1 total:  1.0\n",
      "BACC total:  1.0\n",
      "MCC total:  1.0\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "\n",
    "base_classifier = rf\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_rf'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging with many models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### do bootstrapping "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 1. Multiple subsets are created from the original dataset, selecting observations with replacement.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "\n",
    "num_bootstraps = 10  # Adjust the number of bootstraps as needed\n",
    "\n",
    "original_data_df = X_train_01.assign(label = y_train_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "boot_df = []\n",
    "for i in range(0,num_bootstraps): \n",
    "    boot_df.append(original_data_df.sample(frac = 1, replace=True).reset_index(drop=True))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>ada</th>\n",
       "      <th>svm</th>\n",
       "      <th>cat</th>\n",
       "      <th>xgb</th>\n",
       "      <th>lr</th>\n",
       "      <th>dt</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.994515</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999945</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999321</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.971517</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.556830</td>\n",
       "      <td>0.998437</td>\n",
       "      <td>0.999123</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999128</td>\n",
       "      <td>0.968429</td>\n",
       "      <td>0.361083</td>\n",
       "      <td>0.999123</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.999995</td>\n",
       "      <td>0.999883</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999708</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.880527</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.999996</td>\n",
       "      <td>0.999883</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.559890</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999722</td>\n",
       "      <td>0.974612</td>\n",
       "      <td>0.885473</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.995482</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999675</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999322</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.972065</td>\n",
       "      <td>0.999557</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>0.751246</td>\n",
       "      <td>0.988915</td>\n",
       "      <td>0.995938</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.995480</td>\n",
       "      <td>0.967483</td>\n",
       "      <td>0.990953</td>\n",
       "      <td>0.995955</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>0.751246</td>\n",
       "      <td>0.988915</td>\n",
       "      <td>0.995684</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.995473</td>\n",
       "      <td>0.967483</td>\n",
       "      <td>0.991587</td>\n",
       "      <td>0.995678</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>0.751246</td>\n",
       "      <td>0.988915</td>\n",
       "      <td>0.995938</td>\n",
       "      <td>0.530801</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.995480</td>\n",
       "      <td>0.967483</td>\n",
       "      <td>0.990953</td>\n",
       "      <td>0.995955</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>0.994524</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999929</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999321</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.971521</td>\n",
       "      <td>0.999943</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>0.994513</td>\n",
       "      <td>0.995236</td>\n",
       "      <td>0.999983</td>\n",
       "      <td>0.430217</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.999321</td>\n",
       "      <td>0.971597</td>\n",
       "      <td>0.971515</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>125 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          dnn        rf      lgbm       ada       svm       cat       xgb  \\\n",
       "0    0.994515  0.995236  0.999945  0.430217  0.333333  0.999321  0.971597   \n",
       "1    0.556830  0.998437  0.999123  0.559890  0.333333  0.999128  0.968429   \n",
       "2    0.999995  0.999883  0.999999  0.559890  0.333333  0.999708  0.974612   \n",
       "3    0.999996  0.999883  0.999999  0.559890  0.333333  0.999722  0.974612   \n",
       "4    0.995482  0.995236  0.999675  0.430217  0.333333  0.999322  0.971597   \n",
       "..        ...       ...       ...       ...       ...       ...       ...   \n",
       "120  0.751246  0.988915  0.995938  0.530801  0.333333  0.995480  0.967483   \n",
       "121  0.751246  0.988915  0.995684  0.530801  0.333333  0.995473  0.967483   \n",
       "122  0.751246  0.988915  0.995938  0.530801  0.333333  0.995480  0.967483   \n",
       "123  0.994524  0.995236  0.999929  0.430217  0.333333  0.999321  0.971597   \n",
       "124  0.994513  0.995236  0.999983  0.430217  0.333333  0.999321  0.971597   \n",
       "\n",
       "           lr        dt  label  \n",
       "0    0.971517  1.000000    0.0  \n",
       "1    0.361083  0.999123    1.0  \n",
       "2    0.880527  1.000000    1.0  \n",
       "3    0.885473  1.000000    1.0  \n",
       "4    0.972065  0.999557    0.0  \n",
       "..        ...       ...    ...  \n",
       "120  0.990953  0.995955    2.0  \n",
       "121  0.991587  0.995678    2.0  \n",
       "122  0.990953  0.995955    2.0  \n",
       "123  0.971521  0.999943    0.0  \n",
       "124  0.971515  1.000000    0.0  \n",
       "\n",
       "[125 rows x 10 columns]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boot_df[5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.A base model (weak model) is created on each of these subsets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "bag_comb_pred = []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SVM\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "clf = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "y_train_boot = boot_df[0].pop('label')\n",
    "X_train_boot = boot_df[0]\n",
    "clf.fit(X_train_boot, y_train_boot)\n",
    "preds_svm_01 = clf.predict(X_test_01)\n",
    "bag_comb_pred.append(preds_svm_01)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ADA\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "abc = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "ada = abc.fit(X_train_01, y_train_01)\n",
    "y_train_boot = boot_df[1].pop('label')\n",
    "X_train_boot = boot_df[1]\n",
    "preds_ada_01 = ada.predict(X_test_01)\n",
    "bag_comb_pred.append(preds_ada_01)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 0.9659390\ttest: 0.9677610\tbest: 0.9677610 (0)\ttotal: 1.45ms\tremaining: 143ms\n",
      "10:\tlearn: 0.3667156\ttest: 0.3914904\tbest: 0.3914904 (10)\ttotal: 10.1ms\tremaining: 81.8ms\n",
      "20:\tlearn: 0.1942305\ttest: 0.2349784\tbest: 0.2349784 (20)\ttotal: 19.4ms\tremaining: 73ms\n",
      "30:\tlearn: 0.1179074\ttest: 0.1588624\tbest: 0.1588624 (30)\ttotal: 27.1ms\tremaining: 60.3ms\n",
      "40:\tlearn: 0.0804432\ttest: 0.1209171\tbest: 0.1209171 (40)\ttotal: 33.6ms\tremaining: 48.4ms\n",
      "50:\tlearn: 0.0594240\ttest: 0.1000335\tbest: 0.1000335 (50)\ttotal: 39.4ms\tremaining: 37.8ms\n",
      "60:\tlearn: 0.0458538\ttest: 0.0893526\tbest: 0.0893526 (60)\ttotal: 48.1ms\tremaining: 30.8ms\n",
      "70:\tlearn: 0.0365768\ttest: 0.0827284\tbest: 0.0827284 (70)\ttotal: 54.5ms\tremaining: 22.2ms\n",
      "80:\tlearn: 0.0311332\ttest: 0.0789490\tbest: 0.0789490 (80)\ttotal: 60.7ms\tremaining: 14.2ms\n",
      "90:\tlearn: 0.0267972\ttest: 0.0767081\tbest: 0.0767081 (90)\ttotal: 65.4ms\tremaining: 6.47ms\n",
      "99:\tlearn: 0.0238576\ttest: 0.0760278\tbest: 0.0760278 (99)\ttotal: 69.4ms\tremaining: 0us\n",
      "\n",
      "bestTest = 0.07602778211\n",
      "bestIteration = 99\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Catboost\n",
    "import catboost\n",
    "cat_01 = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "y_train_boot = boot_df[2].pop('label')\n",
    "X_train_boot = boot_df[2]\n",
    "cat_01.fit(X_train_boot, y_train_boot, eval_set=(X_test_01, y_test_01), verbose=10)\n",
    "preds_cat = cat_01.predict(X_test_01)\n",
    "preds_cat = np.squeeze(preds_cat)\n",
    "pred_label = preds_cat\n",
    "bag_comb_pred.append(preds_cat)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n"
     ]
    }
   ],
   "source": [
    "#MLP\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "y_train_boot = boot_df[3].pop('label')\n",
    "X_train_boot = boot_df[3]\n",
    "if 1 == 1 and 0 == 0:\n",
    "    MLP = mlp.fit(X_train_boot, y_train_boot)\n",
    "    y_pred = MLP.predict_proba(X_test_01)\n",
    "    preds_mlp_01 = np.argmax(y_pred,axis = 1)\n",
    "\n",
    "bag_comb_pred.append(preds_mlp_01)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining LGBM Model\n",
      "---------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#LGBM\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining LGBM Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#LGBM\n",
    "from lightgbm import LGBMClassifier\n",
    "lgbm = LGBMClassifier()\n",
    "y_train_boot = boot_df[4].pop('label')\n",
    "X_train_boot = boot_df[4]\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "    lgbm.fit(X_train_boot, y_train_boot)\n",
    "    preds_lgbm_01 = lgbm.predict(X_test_01)\n",
    "    bag_comb_pred.append(preds_lgbm_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "#KNN\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_clf_01=KNeighborsClassifier(n_neighbors = 5)\n",
    "y_train_boot = boot_df[5].pop('label')\n",
    "X_train_boot = boot_df[5]\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "    knn_clf_01.fit(X_train_boot,y_train_boot)\n",
    "if use_model_knn == 1:\n",
    "    preds_knn =knn_clf_01.predict(X_test_01)\n",
    "    bag_comb_pred.append(preds_knn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Random Forest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "y_train_boot = boot_df[6].pop('label')\n",
    "X_train_boot = boot_df[6]\n",
    "\n",
    "if True == True:\n",
    "    model_rf_01 = rf.fit(X_train_boot,y_train_boot)\n",
    "    preds_rf_01 = model_rf_01.predict(X_test_01)\n",
    "    bag_comb_pred.append(preds_rf_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "1/1 [==============================] - 1s 540ms/step - loss: 1.5954 - accuracy: 0.3000 - val_loss: 1.5868 - val_accuracy: 0.3600\n",
      "Epoch 2/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 1.5931 - accuracy: 0.2700 - val_loss: 1.5826 - val_accuracy: 0.3600\n",
      "Epoch 3/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 1.5893 - accuracy: 0.3200 - val_loss: 1.5782 - val_accuracy: 0.4000\n",
      "Epoch 4/100\n",
      "1/1 [==============================] - 0s 15ms/step - loss: 1.5898 - accuracy: 0.3300 - val_loss: 1.5739 - val_accuracy: 0.4000\n",
      "Epoch 5/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 1.5909 - accuracy: 0.3600 - val_loss: 1.5696 - val_accuracy: 0.4000\n",
      "Epoch 6/100\n",
      "1/1 [==============================] - 0s 30ms/step - loss: 1.5875 - accuracy: 0.3300 - val_loss: 1.5652 - val_accuracy: 0.4000\n",
      "Epoch 7/100\n",
      "1/1 [==============================] - 0s 25ms/step - loss: 1.5752 - accuracy: 0.3400 - val_loss: 1.5607 - val_accuracy: 0.4000\n",
      "Epoch 8/100\n",
      "1/1 [==============================] - 0s 30ms/step - loss: 1.5760 - accuracy: 0.3400 - val_loss: 1.5563 - val_accuracy: 0.4000\n",
      "Epoch 9/100\n",
      "1/1 [==============================] - 0s 27ms/step - loss: 1.5665 - accuracy: 0.3400 - val_loss: 1.5517 - val_accuracy: 0.4000\n",
      "Epoch 10/100\n",
      "1/1 [==============================] - 0s 21ms/step - loss: 1.5782 - accuracy: 0.3300 - val_loss: 1.5472 - val_accuracy: 0.4000\n",
      "Epoch 11/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 1.5615 - accuracy: 0.3300 - val_loss: 1.5426 - val_accuracy: 0.4000\n",
      "Epoch 12/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 1.5515 - accuracy: 0.3300 - val_loss: 1.5380 - val_accuracy: 0.3600\n",
      "Epoch 13/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 1.5460 - accuracy: 0.3100 - val_loss: 1.5332 - val_accuracy: 0.3600\n"
     ]
    }
   ],
   "source": [
    "#DNN\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "#Model Parameters\n",
    "y_train_boot = boot_df[7].pop('label')\n",
    "X_train_boot = boot_df[7]\n",
    "\n",
    "\n",
    "dropout_rate = 0.02\n",
    "nodes = 3\n",
    "out_layer = 5\n",
    "optimizer='adam'\n",
    "loss='sparse_categorical_crossentropy'\n",
    "epochs=100\n",
    "batch_size=128\n",
    "num_columns = X_train_boot.shape[1]\n",
    "dnn_01 = tf.keras.Sequential()\n",
    "# Input layer\n",
    "dnn_01.add(tf.keras.Input(shape=(num_columns,)))\n",
    "# Dense layers with dropout\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "# Output layer\n",
    "# dnn_01.add(tf.keras.layers.Dense(out_layer))\n",
    "dnn_01.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "# dnn.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "dnn_01.compile(optimizer=optimizer, loss=loss,metrics=['accuracy'])\n",
    "from keras.callbacks import EarlyStopping\n",
    "# Define EarlyStopping callback\n",
    "early_stopping = EarlyStopping(monitor='val_accuracy', patience=10, restore_best_weights=True)\n",
    "dnn_01.fit(X_train_boot, y_train_boot, epochs=epochs, batch_size=batch_size,validation_split=0.2, callbacks=[early_stopping])\n",
    "pred_dnn = dnn_01.predict(X_test_01)\n",
    "preds_dnn_01 = np.argmax(pred_dnn,axis = 1)\n",
    "bag_comb_pred.append(preds_dnn_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "#LogReg\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "logreg_01 = LogisticRegression()\n",
    "y_train_boot = boot_df[8].pop('label')\n",
    "X_train_boot = boot_df[8]\n",
    "\n",
    "logreg_01.fit(X_train_boot,y_train_boot)\n",
    "preds_logreg =logreg_01.predict(X_test_01)\n",
    "bag_comb_pred.append(preds_logreg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "y_train_boot = boot_df[9].pop('label')\n",
    "X_train_boot = boot_df[9]\n",
    "\n",
    "# Create a DMatrix for XGBoost\n",
    "dtrain = xgb.DMatrix(X_train_boot, label=y_train_boot)\n",
    "dtest = xgb.DMatrix(X_test_01, label=y_test_01)\n",
    "# Set XGBoost parameters\n",
    "params = {\n",
    "    'objective': 'multi:softmax',  # for multi-class classification\n",
    "    'num_class': 5,  # specify the number of classes\n",
    "    'max_depth': 3,\n",
    "    'learning_rate': 0.1,\n",
    "    'eval_metric': 'mlogloss'  # metric for multi-class classification\n",
    "}\n",
    "# Train the XGBoost model\n",
    "num_round = 100\n",
    "xgb_01 = xgb.train(params, dtrain, num_round)\n",
    "preds_xgb_01 = xgb_01.predict(dtest)\n",
    "bag_comb_pred.append(preds_xgb_01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. The models run in parallel and are independent of each other."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    model_0  model_1  model_2  model_3  model_4  model_5  model_6  model_7  \\\n",
      "0       0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "1       2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "2       2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "3       0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "4       2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "5       2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "6       1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "7       2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "8       2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "9       0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "10      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "11      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "12      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "13      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "14      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "15      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "16      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "17      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "18      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "19      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "20      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "21      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "22      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "23      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "24      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "25      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "26      0.0      1.0      1.0        0      1.0      1.0      1.0        1   \n",
      "27      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "28      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "29      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "30      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "31      1.0      1.0      0.0        1      1.0      1.0      1.0        0   \n",
      "32      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "33      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "34      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "35      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "36      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "37      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "38      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "39      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "40      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "41      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "42      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "43      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "44      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "45      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "46      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "47      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "48      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "49      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "50      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "51      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "52      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "53      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "54      0.0      1.0      0.0        0      0.0      1.0      1.0        0   \n",
      "\n",
      "    model_8  model_9  \n",
      "0       0.0      0.0  \n",
      "1       2.0      2.0  \n",
      "2       2.0      2.0  \n",
      "3       0.0      0.0  \n",
      "4       2.0      2.0  \n",
      "5       2.0      2.0  \n",
      "6       1.0      1.0  \n",
      "7       2.0      2.0  \n",
      "8       2.0      2.0  \n",
      "9       0.0      0.0  \n",
      "10      0.0      0.0  \n",
      "11      1.0      1.0  \n",
      "12      2.0      2.0  \n",
      "13      1.0      1.0  \n",
      "14      0.0      0.0  \n",
      "15      1.0      1.0  \n",
      "16      1.0      1.0  \n",
      "17      0.0      0.0  \n",
      "18      1.0      1.0  \n",
      "19      1.0      1.0  \n",
      "20      1.0      1.0  \n",
      "21      0.0      0.0  \n",
      "22      1.0      1.0  \n",
      "23      2.0      2.0  \n",
      "24      1.0      1.0  \n",
      "25      0.0      0.0  \n",
      "26      1.0      1.0  \n",
      "27      0.0      0.0  \n",
      "28      0.0      0.0  \n",
      "29      0.0      0.0  \n",
      "30      1.0      1.0  \n",
      "31      1.0      1.0  \n",
      "32      2.0      2.0  \n",
      "33      2.0      2.0  \n",
      "34      0.0      0.0  \n",
      "35      2.0      2.0  \n",
      "36      1.0      1.0  \n",
      "37      2.0      2.0  \n",
      "38      0.0      0.0  \n",
      "39      1.0      1.0  \n",
      "40      2.0      2.0  \n",
      "41      0.0      0.0  \n",
      "42      0.0      0.0  \n",
      "43      1.0      1.0  \n",
      "44      0.0      0.0  \n",
      "45      0.0      0.0  \n",
      "46      2.0      2.0  \n",
      "47      1.0      1.0  \n",
      "48      2.0      2.0  \n",
      "49      0.0      0.0  \n",
      "50      0.0      0.0  \n",
      "51      2.0      2.0  \n",
      "52      1.0      1.0  \n",
      "53      0.0      0.0  \n",
      "54      0.0      1.0  \n"
     ]
    }
   ],
   "source": [
    "bag_vot_df = pd.DataFrame()\n",
    "for i in range(0,len(bag_comb_pred)):\n",
    "    bag_vot_df[f'model_{i}'] =  bag_comb_pred[i]\n",
    "print(bag_vot_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    model_0  model_1  model_2  model_3  model_4  model_5  model_6  model_7  \\\n",
      "0       0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "1       2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "2       2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "3       0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "4       2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "5       2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "6       1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "7       2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "8       2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "9       0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "10      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "11      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "12      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "13      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "14      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "15      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "16      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "17      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "18      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "19      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "20      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "21      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "22      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "23      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "24      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "25      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "26      0.0      1.0      1.0        0      1.0      1.0      1.0        1   \n",
      "27      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "28      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "29      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "30      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "31      1.0      1.0      0.0        1      1.0      1.0      1.0        0   \n",
      "32      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "33      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "34      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "35      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "36      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "37      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "38      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "39      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "40      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "41      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "42      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "43      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "44      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "45      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "46      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "47      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "48      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "49      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "50      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "51      2.0      2.0      2.0        2      2.0      2.0      2.0        0   \n",
      "52      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "53      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "54      0.0      1.0      0.0        0      0.0      1.0      1.0        0   \n",
      "\n",
      "    model_8  model_9  ensemble  \n",
      "0       0.0      0.0         0  \n",
      "1       2.0      2.0         2  \n",
      "2       2.0      2.0         2  \n",
      "3       0.0      0.0         0  \n",
      "4       2.0      2.0         2  \n",
      "5       2.0      2.0         2  \n",
      "6       1.0      1.0         1  \n",
      "7       2.0      2.0         2  \n",
      "8       2.0      2.0         2  \n",
      "9       0.0      0.0         0  \n",
      "10      0.0      0.0         0  \n",
      "11      1.0      1.0         1  \n",
      "12      2.0      2.0         2  \n",
      "13      1.0      1.0         1  \n",
      "14      0.0      0.0         0  \n",
      "15      1.0      1.0         1  \n",
      "16      1.0      1.0         1  \n",
      "17      0.0      0.0         0  \n",
      "18      1.0      1.0         1  \n",
      "19      1.0      1.0         1  \n",
      "20      1.0      1.0         1  \n",
      "21      0.0      0.0         0  \n",
      "22      1.0      1.0         1  \n",
      "23      2.0      2.0         2  \n",
      "24      1.0      1.0         1  \n",
      "25      0.0      0.0         0  \n",
      "26      1.0      1.0         1  \n",
      "27      0.0      0.0         0  \n",
      "28      0.0      0.0         0  \n",
      "29      0.0      0.0         0  \n",
      "30      1.0      1.0         1  \n",
      "31      1.0      1.0         1  \n",
      "32      2.0      2.0         2  \n",
      "33      2.0      2.0         2  \n",
      "34      0.0      0.0         0  \n",
      "35      2.0      2.0         2  \n",
      "36      1.0      1.0         1  \n",
      "37      2.0      2.0         2  \n",
      "38      0.0      0.0         0  \n",
      "39      1.0      1.0         1  \n",
      "40      2.0      2.0         2  \n",
      "41      0.0      0.0         0  \n",
      "42      0.0      0.0         0  \n",
      "43      1.0      1.0         1  \n",
      "44      0.0      0.0         0  \n",
      "45      0.0      0.0         0  \n",
      "46      2.0      2.0         2  \n",
      "47      1.0      1.0         1  \n",
      "48      2.0      2.0         2  \n",
      "49      0.0      0.0         0  \n",
      "50      0.0      0.0         0  \n",
      "51      2.0      2.0         2  \n",
      "52      1.0      1.0         1  \n",
      "53      0.0      0.0         0  \n",
      "54      0.0      1.0         0  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0     0\n",
       "1     2\n",
       "2     2\n",
       "3     0\n",
       "4     2\n",
       "5     2\n",
       "6     1\n",
       "7     2\n",
       "8     2\n",
       "9     0\n",
       "10    0\n",
       "11    1\n",
       "12    2\n",
       "13    1\n",
       "14    0\n",
       "15    1\n",
       "16    1\n",
       "17    0\n",
       "18    1\n",
       "19    1\n",
       "20    1\n",
       "21    0\n",
       "22    1\n",
       "23    2\n",
       "24    1\n",
       "25    0\n",
       "26    1\n",
       "27    0\n",
       "28    0\n",
       "29    0\n",
       "30    1\n",
       "31    1\n",
       "32    2\n",
       "33    2\n",
       "34    0\n",
       "35    2\n",
       "36    1\n",
       "37    2\n",
       "38    0\n",
       "39    1\n",
       "40    2\n",
       "41    0\n",
       "42    0\n",
       "43    1\n",
       "44    0\n",
       "45    0\n",
       "46    2\n",
       "47    1\n",
       "48    2\n",
       "49    0\n",
       "50    0\n",
       "51    2\n",
       "52    1\n",
       "53    0\n",
       "54    0\n",
       "Name: ensemble, dtype: int64"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Voting start\n",
    "from scipy.stats import mode\n",
    "# bag_comb_pred_df = pd.DataFrame(bag_comb_pred)\n",
    "# Extract predictions columns\n",
    "\n",
    "# predictions = df[['dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm','cat','xgb']]\n",
    "    # selected_columns = df.loc[:, ~df.columns.isin(['rf'])]\n",
    "predictions = bag_vot_df \n",
    "\n",
    "# predictions = bag_comb_pred_df.loc[:, ~df.columns.isin(['label'])] #df[column_features]\n",
    "\n",
    "# Use the mode function along axis 1 to get the most common prediction for each row\n",
    "ensemble_predictions, _ = mode(predictions.values, axis=1)\n",
    "\n",
    "# Add the ensemble predictions to the DataFrame\n",
    "bag_vot_df['ensemble'] = ensemble_predictions.astype(int)\n",
    "\n",
    "# Display the DataFrame with ensemble predictions\n",
    "print(bag_vot_df)\n",
    "\n",
    "pred_label = bag_vot_df ['ensemble'].values\n",
    "bag_vot_df.pop('ensemble')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0     1     2\n",
      "0  20.0   0.0   0.0\n",
      "1   1.0  18.0   0.0\n",
      "2   0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9818181818181818\n",
      "Precision total:  0.9841269841269842\n",
      "Recall total:  0.9824561403508771\n",
      "F1 total:  0.9828609096901779\n",
      "BACC total:  0.9824561403508771\n",
      "MCC total:  0.973081241361401\n"
     ]
    }
   ],
   "source": [
    "name='bag_comb'\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining DNN Model\n",
      "---------------------------------------------------------------------------------\n",
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_6 (Dense)              (None, 3)                 30        \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_6 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_7 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_8 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_9 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 5)                 20        \n",
      "=================================================================\n",
      "Total params: 98\n",
      "Trainable params: 98\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining DNN Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_dnn = time.time()\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "#Model Parameters\n",
    "dropout_rate = 0.2\n",
    "nodes = 3\n",
    "out_layer = 5\n",
    "optimizer='adam'\n",
    "loss='sparse_categorical_crossentropy'\n",
    "epochs=100\n",
    "batch_size=128\n",
    "\n",
    "\n",
    "num_columns = X_train_01.shape[1]\n",
    "\n",
    "dnn_01 = tf.keras.Sequential()\n",
    "\n",
    "# Input layer\n",
    "dnn_01.add(tf.keras.Input(shape=(num_columns,)))\n",
    "\n",
    "# # Dense layers with dropout\n",
    "# dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(2*nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(3*nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(2*nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "\n",
    "\n",
    "# Dense layers with dropout\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# Output layer\n",
    "# dnn_01.add(tf.keras.layers.Dense(out_layer))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "# dnn.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "\n",
    "\n",
    "dnn_01.compile(optimizer=optimizer, loss=loss,metrics=['accuracy'])\n",
    "\n",
    "dnn_01.summary()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Training DNN\n",
      "---------------------------------------------------------------------------------\n",
      "Epoch 1/100\n",
      "1/1 [==============================] - 0s 393ms/step - loss: 2.5978 - accuracy: 0.0800 - val_loss: 1.7692 - val_accuracy: 0.0000e+00\n",
      "Epoch 2/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 2.6232 - accuracy: 0.1100 - val_loss: 1.7365 - val_accuracy: 0.0000e+00\n",
      "Epoch 3/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 2.4576 - accuracy: 0.1000 - val_loss: 1.7069 - val_accuracy: 0.0000e+00\n",
      "Epoch 4/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 2.4550 - accuracy: 0.1700 - val_loss: 1.6792 - val_accuracy: 0.0000e+00\n",
      "Epoch 5/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 2.0916 - accuracy: 0.1300 - val_loss: 1.6526 - val_accuracy: 0.0000e+00\n",
      "Epoch 6/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 2.1564 - accuracy: 0.2100 - val_loss: 1.6262 - val_accuracy: 0.0000e+00\n",
      "Epoch 7/100\n",
      "1/1 [==============================] - 0s 15ms/step - loss: 2.1444 - accuracy: 0.1900 - val_loss: 1.6016 - val_accuracy: 0.0000e+00\n",
      "Epoch 8/100\n",
      "1/1 [==============================] - 0s 14ms/step - loss: 2.2982 - accuracy: 0.1300 - val_loss: 1.5780 - val_accuracy: 0.4800\n",
      "Epoch 9/100\n",
      "1/1 [==============================] - 0s 19ms/step - loss: 2.3026 - accuracy: 0.1300 - val_loss: 1.5563 - val_accuracy: 0.4800\n",
      "Epoch 10/100\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 2.3364 - accuracy: 0.1600 - val_loss: 1.5359 - val_accuracy: 0.4800\n",
      "Epoch 11/100\n",
      "1/1 [==============================] - 0s 15ms/step - loss: 2.2323 - accuracy: 0.1800 - val_loss: 1.5168 - val_accuracy: 0.4800\n",
      "Epoch 12/100\n",
      "1/1 [==============================] - 0s 15ms/step - loss: 2.0243 - accuracy: 0.1800 - val_loss: 1.4993 - val_accuracy: 0.4800\n",
      "Epoch 13/100\n",
      "1/1 [==============================] - 0s 31ms/step - loss: 2.0000 - accuracy: 0.2500 - val_loss: 1.4827 - val_accuracy: 0.4800\n",
      "Epoch 14/100\n",
      "1/1 [==============================] - 0s 26ms/step - loss: 2.0323 - accuracy: 0.1700 - val_loss: 1.4678 - val_accuracy: 0.4800\n",
      "Epoch 15/100\n",
      "1/1 [==============================] - 0s 19ms/step - loss: 2.0595 - accuracy: 0.2400 - val_loss: 1.4544 - val_accuracy: 0.4800\n",
      "Epoch 16/100\n",
      "1/1 [==============================] - 0s 17ms/step - loss: 2.0843 - accuracy: 0.2100 - val_loss: 1.4420 - val_accuracy: 0.4800\n",
      "Epoch 17/100\n",
      "1/1 [==============================] - 0s 19ms/step - loss: 1.9659 - accuracy: 0.1900 - val_loss: 1.4302 - val_accuracy: 0.4800\n",
      "Epoch 18/100\n",
      "1/1 [==============================] - 0s 33ms/step - loss: 1.7385 - accuracy: 0.2700 - val_loss: 1.4195 - val_accuracy: 0.4800\n"
     ]
    }
   ],
   "source": [
    "#DNN\n",
    "try:\n",
    "    from keras.callbacks import EarlyStopping\n",
    "\n",
    "    # Define EarlyStopping callback\n",
    "    early_stopping = EarlyStopping(monitor='val_accuracy', patience=10, restore_best_weights=True)\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training DNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training DNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    # Convert Y_test back to its original format\n",
    "    # y_test = np.argmax(Y_test, axis=1)\n",
    "\n",
    "    # Start the timer\n",
    "    start = time.time()\n",
    "    # dnn_01.fit(X_train_01, y_train_01, epochs=epochs, batch_size=batch_size)\n",
    "    dnn_01.fit(X_train_01, y_train_01, epochs=epochs, batch_size=batch_size,validation_split=0.2, callbacks=[early_stopping])\n",
    "\n",
    "    # model.fit(x_train, Y_train, epochs=100, batch_size=128, validation_split=0.2, callbacks=[early_stopping])\n",
    "\n",
    "    # End the timer\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    # joblib.dump(dnn_01, 'dnn_level_01.joblib')\n",
    "    # dnn_01.save(\"dnn_level_01.h5\")\n",
    "\n",
    "    # Calculate the time taken and print it out\n",
    "    # print(f'Time taken for training: {time_taken} seconds')\n",
    "except: \n",
    "    None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dnn_01 = load_model(\"dnn_level_01.h5\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DNN\n",
    "try:\n",
    "    start = time.time()\n",
    "    pred_dnn = dnn_01.predict(X_test_01)\n",
    "    preds_dnn_01 = np.argmax(pred_dnn,axis = 1)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "except:\n",
    "        with open(output_file_name, \"a\") as f: print('error', file = f)\n",
    "        preds_dnn_01 = 0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0  1.0  2.0  3.0\n",
      "0.0  20.0  0.0  0.0  0.0\n",
      "1.0  16.0  3.0  0.0  0.0\n",
      "2.0  16.0  0.0  0.0  0.0\n",
      "3.0   0.0  0.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.36363636363636365\n",
      "Precision total:  0.09615384615384616\n",
      "Recall total:  0.25\n",
      "F1 total:  0.1388888888888889\n",
      "BACC total:  0.3333333333333333\n",
      "MCC total:  0.07580399584511639\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "y_pred contains classes not in y_true\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    name = 'dnn'\n",
    "    pred_label = preds_dnn_01\n",
    "        \n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_dnn\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n",
    "except: None    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining SVM Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#SVM\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining SVM Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_svm = time.time()\n",
    "\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "# Instantiate the SGDClassifier with additional hyperparameters\n",
    "clf = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "\n",
    "#SVM\n",
    "start = time.time()\n",
    "clf.fit(X_train_01, y_train_01)\n",
    "end = time.time()\n",
    "clf.score(X_train_01, y_train_01)\n",
    "time_taken = end - start\n",
    "with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "joblib.dump(clf, 'svm_level_01.joblib')\n",
    "\n",
    "\n",
    "clf = loaded_model = joblib.load('svm_level_01.joblib')\n",
    "\n",
    "\n",
    "#SVM\n",
    "start = time.time()\n",
    "preds_svm_01 = clf.predict(X_test_01)\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0   1.0   2.0\n",
      "0.0  20.0   0.0   0.0\n",
      "1.0   2.0  17.0   0.0\n",
      "2.0   0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9636363636363636\n",
      "Precision total:  0.9696969696969697\n",
      "Recall total:  0.9649122807017544\n",
      "F1 total:  0.9656084656084656\n",
      "BACC total:  0.9649122807017544\n",
      "MCC total:  0.9470572003091526\n"
     ]
    }
   ],
   "source": [
    "\n",
    "pred_label = preds_svm_01\n",
    "name = 'svm'\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start_svm\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining RF Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training RF\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction RF\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0   1.0   2.0\n",
      "0.0  20.0   0.0   0.0\n",
      "1.0   0.0  19.0   0.0\n",
      "2.0   0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  1.0\n",
      "Precision total:  1.0\n",
      "Recall total:  1.0\n",
      "F1 total:  1.0\n",
      "BACC total:  1.0\n",
      "MCC total:  1.0\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining RF Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_rf = time.time()\n",
    "\n",
    "#Random Forest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "#------------------------------------------------------------------------------\n",
    "\n",
    "if True == True:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training RF')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Training RF', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #RF\n",
    "    start = time.time()\n",
    "    model_rf_01 = rf.fit(X_train_01,y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(model_rf_01, X_train_01, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(model_rf_01, 'rf_base_model_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "    model_rf_01  = joblib.load('rf_base_model_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Prediction RF')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction RF', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #RF\n",
    "    start = time.time()\n",
    "    preds_rf_01 = model_rf_01.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('-------------------------------------------------------', file = f)\n",
    "pred_label = preds_rf_01\n",
    "name='rf'\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start_rf\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining LGBM Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training LGBM\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction LGBM\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0   1.0   2.0\n",
      "0.0  20.0   0.0   0.0\n",
      "1.0   0.0  19.0   0.0\n",
      "2.0   0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  1.0\n",
      "Precision total:  1.0\n",
      "Recall total:  1.0\n",
      "F1 total:  1.0\n",
      "BACC total:  1.0\n",
      "MCC total:  1.0\n"
     ]
    }
   ],
   "source": [
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining LGBM Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#LGBM\n",
    "from lightgbm import LGBMClassifier\n",
    "lgbm = LGBMClassifier()\n",
    "\n",
    "start_lgbm = time.time()\n",
    "\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training LGBM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training LGBM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    lgbm.fit(X_train_01, y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(lgbm, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(lgbm, 'lgbm_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "    lgbm = joblib.load('lgbm_01.joblib')\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    print('Prediction LGBM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction LGBM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #LGBM\n",
    "    start = time.time()\n",
    "    preds_lgbm_01 = lgbm.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    pred_label = preds_lgbm_01\n",
    "    name='lgbm'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_lgbm\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining MLP Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training MLP\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0     1     2\n",
      "0  20.0   0.0   0.0\n",
      "1   3.0  16.0   0.0\n",
      "2   0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9454545454545454\n",
      "Precision total:  0.9565217391304347\n",
      "Recall total:  0.9473684210526315\n",
      "F1 total:  0.9481727574750831\n",
      "BACC total:  0.9473684210526315\n",
      "MCC total:  0.9218603475851501\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#MLP\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining MLP Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_mlp = time.time()\n",
    "\n",
    "\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "import time\n",
    "\n",
    "# create MLPClassifier instance\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training MLP')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Training MLP', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    start = time.time()\n",
    "    MLP = mlp.fit(X_train_01, y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(MLP, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(MLP, 'mlp_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "    MLP = joblib.load('mlp_01.joblib')\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    #MLP\n",
    "    start = time.time()\n",
    "    y_pred = MLP.predict_proba(X_test_01)\n",
    "    preds_mlp_01 = np.argmax(y_pred,axis = 1)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#MLP\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('MLP 01 model', file = f)\n",
    "    pred_label = preds_mlp_01\n",
    "    name='mlp'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_mlp\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ADA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining ADA Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training ADA\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction ADA\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0   1.0   2.0\n",
      "0.0  20.0   0.0   0.0\n",
      "1.0   0.0  19.0   0.0\n",
      "2.0   0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  1.0\n",
      "Precision total:  1.0\n",
      "Recall total:  1.0\n",
      "F1 total:  1.0\n",
      "BACC total:  1.0\n",
      "MCC total:  1.0\n"
     ]
    }
   ],
   "source": [
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining ADA Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#ADA\n",
    "# from sklearn.multioutput import MultiOutputClassifier\n",
    "start_ada = time.time()\n",
    "\n",
    "\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "import time\n",
    "abc = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training ADA')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training ADA', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #ADA\n",
    "\n",
    "\n",
    "    start = time.time()\n",
    "    ada = abc.fit(X_train_01, y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(ada, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "\n",
    "    # Assuming 'model' is your trained model\n",
    "    joblib.dump(ada, 'ada_01.joblib')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "    ada = joblib.load('ada_01.joblib')\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    print('Prediction ADA')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction ADA', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #ADA\n",
    "    start = time.time()\n",
    "    preds_ada_01 = ada.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('ADA 01 model', file = f)\n",
    "\n",
    "\n",
    "    pred_label = preds_ada_01\n",
    "    name='ada'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_ada\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining KNN Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training KNN\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0   1.0   2.0\n",
      "0.0  20.0   0.0   0.0\n",
      "1.0   0.0  19.0   0.0\n",
      "2.0   0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  1.0\n",
      "Precision total:  1.0\n",
      "Recall total:  1.0\n",
      "F1 total:  1.0\n",
      "BACC total:  1.0\n",
      "MCC total:  1.0\n"
     ]
    }
   ],
   "source": [
    "#KNN\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining KNN Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_knn = time.time()\n",
    "\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_clf_01=KNeighborsClassifier(n_neighbors = 5)\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "    #KNN\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training KNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training KNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    knn_clf_01.fit(X_train_01,y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(knn_clf, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(knn_clf_01, 'knn_01.joblib')\n",
    "\n",
    "\n",
    "if load_model_knn == 1:\n",
    "    knn_clf_01 = joblib.load('knn_01.joblib')\n",
    "\n",
    "if use_model_knn == 1:\n",
    "\n",
    "    #KNN\n",
    "    start = time.time()\n",
    "    preds_knn =knn_clf_01.predict(X_test_01)\n",
    "    preds_knn\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('KNN 01 model', file = f)\n",
    "\n",
    "    pred_label = preds_knn\n",
    "    name='knn'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start_knn\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Log Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining Logistic Regression Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training LR \n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0   1.0   2.0\n",
      "0.0  20.0   0.0   0.0\n",
      "1.0   4.0  15.0   0.0\n",
      "2.0   0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9272727272727272\n",
      "Precision total:  0.9444444444444445\n",
      "Recall total:  0.9298245614035089\n",
      "F1 total:  0.93048128342246\n",
      "BACC total:  0.9298245614035089\n",
      "MCC total:  0.8974297218132932\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#Logistic Regression\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining Logistic Regression Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "logreg_01 = LogisticRegression()\n",
    "start_lr = time.time()\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "    #KNN\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training LR ')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training LR', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    logreg_01.fit(X_train_01,y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(knn_clf, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(logreg_01, 'logreg_01.joblib')\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "    logreg_01 = joblib.load('logreg_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    #lR\n",
    "    start = time.time()\n",
    "    preds_logreg =logreg_01.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#LR\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('LR 01 model', file = f)\n",
    "\n",
    "    pred_label = preds_logreg\n",
    "    # pred_label = label[ypred]\n",
    "    name='lr'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_lr\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 0.9758843\ttest: 0.9665396\tbest: 0.9665396 (0)\ttotal: 1.25ms\tremaining: 124ms\n",
      "10:\tlearn: 0.4088649\ttest: 0.3723855\tbest: 0.3723855 (10)\ttotal: 9.7ms\tremaining: 78.5ms\n",
      "20:\tlearn: 0.2241960\ttest: 0.1949047\tbest: 0.1949047 (20)\ttotal: 19.1ms\tremaining: 71.9ms\n",
      "30:\tlearn: 0.1451259\ttest: 0.1202189\tbest: 0.1202189 (30)\ttotal: 28.4ms\tremaining: 63.2ms\n",
      "40:\tlearn: 0.1024065\ttest: 0.0812747\tbest: 0.0812747 (40)\ttotal: 36.8ms\tremaining: 53ms\n",
      "50:\tlearn: 0.0767395\ttest: 0.0599626\tbest: 0.0599626 (50)\ttotal: 45ms\tremaining: 43.3ms\n",
      "60:\tlearn: 0.0604420\ttest: 0.0480502\tbest: 0.0480502 (60)\ttotal: 51.7ms\tremaining: 33ms\n",
      "70:\tlearn: 0.0476017\ttest: 0.0392048\tbest: 0.0392048 (70)\ttotal: 57.4ms\tremaining: 23.4ms\n",
      "80:\tlearn: 0.0397129\ttest: 0.0339498\tbest: 0.0339498 (80)\ttotal: 62.7ms\tremaining: 14.7ms\n",
      "90:\tlearn: 0.0336844\ttest: 0.0298997\tbest: 0.0298997 (90)\ttotal: 67.1ms\tremaining: 6.64ms\n",
      "99:\tlearn: 0.0294444\ttest: 0.0274367\tbest: 0.0274367 (99)\ttotal: 71.1ms\tremaining: 0us\n",
      "\n",
      "bestTest = 0.02743670757\n",
      "bestIteration = 99\n",
      "\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0   1.0   2.0\n",
      "0.0  20.0   0.0   0.0\n",
      "1.0   0.0  19.0   0.0\n",
      "2.0   0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  1.0\n",
      "Precision total:  1.0\n",
      "Recall total:  1.0\n",
      "F1 total:  1.0\n",
      "BACC total:  1.0\n",
      "MCC total:  1.0\n"
     ]
    }
   ],
   "source": [
    "import catboost\n",
    "start = time.time()\n",
    "\n",
    "cat_01 = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "\n",
    "# Fit the model\n",
    "cat_01.fit(X_train_01, y_train_01, eval_set=(X_test_01, y_test_01), verbose=10)\n",
    "\n",
    "# Make predictions on the test set\n",
    "preds_cat = cat_01.predict(X_test_01)\n",
    "preds_cat = np.squeeze(preds_cat)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('catboost', file = f)\n",
    "\n",
    "\n",
    "pred_label = preds_cat\n",
    "name='cat'\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0   1.0   2.0\n",
      "0.0  20.0   0.0   0.0\n",
      "1.0   0.0  19.0   0.0\n",
      "2.0   0.0   0.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  1.0\n",
      "Precision total:  1.0\n",
      "Recall total:  1.0\n",
      "F1 total:  1.0\n",
      "BACC total:  1.0\n",
      "MCC total:  1.0\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import xgboost as xgb\n",
    "start = time.time()\n",
    "\n",
    "# Create a DMatrix for XGBoost\n",
    "dtrain = xgb.DMatrix(X_train_01, label=y_train_01)\n",
    "dtest = xgb.DMatrix(X_test_01, label=y_test_01)\n",
    "\n",
    "# Set XGBoost parameters\n",
    "params = {\n",
    "    'objective': 'multi:softmax',  # for multi-class classification\n",
    "    'num_class': 5,  # specify the number of classes\n",
    "    'max_depth': 3,\n",
    "    'learning_rate': 0.1,\n",
    "    'eval_metric': 'mlogloss'  # metric for multi-class classification\n",
    "}\n",
    "\n",
    "# Train the XGBoost model\n",
    "num_round = 100\n",
    "xgb_01 = xgb.train(params, dtrain, num_round)\n",
    "\n",
    "# Make predictions on the test set\n",
    "preds_xgb_01 = xgb_01.predict(dtest)\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('xgboost base model', file = f)\n",
    "\n",
    "    pred_label = preds_xgb_01\n",
    "    name='xgb'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ### Generating Summary Metric Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+----------+-----------+----------+----------+\n",
      "| Models      |   ACC-01 |    PRE-01 |   REC-01 |    F1-01 |\n",
      "+=============+==========+===========+==========+==========+\n",
      "| ADA         | 1        | 1         | 1        | 1        |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| KNN         | 1        | 1         | 1        | 1        |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| CAT         | 1        | 1         | 1        | 1        |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| XGB         | 1        | 1         | 1        | 1        |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| LGBM        | 1        | 1         | 1        | 1        |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| RF          | 1        | 1         | 1        | 1        |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| Bag_knn     | 1        | 1         | 1        | 1        |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| Bag_rf      | 1        | 1         | 1        | 1        |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| Bag_cat     | 1        | 1         | 1        | 1        |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| Bag_DT      | 0.981818 | 0.984127  | 0.982456 | 0.982861 |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| Bag_ada     | 0.981818 | 0.984127  | 0.982456 | 0.982861 |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| Bag_lgbm    | 0.981818 | 0.984127  | 0.982456 | 0.982861 |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| Bag_comb    | 0.981818 | 0.984127  | 0.982456 | 0.982861 |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| SVM         | 0.963636 | 0.969697  | 0.964912 | 0.965608 |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| Bag_LR      | 0.963636 | 0.969697  | 0.964912 | 0.965608 |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| Bag_mlp     | 0.963636 | 0.969697  | 0.964912 | 0.965608 |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| MLP         | 0.945455 | 0.956522  | 0.947368 | 0.948173 |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| LR          | 0.927273 | 0.944444  | 0.929825 | 0.930481 |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| Bag_svm     | 0.618182 | 0.491228  | 0.649123 | 0.54386  |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| avg         | 0.345455 | 0.115152  | 0.333333 | 0.171171 |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| weighed_avg | 0.345455 | 0.115152  | 0.333333 | 0.171171 |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| DNN         | 0.363636 | 0.0961538 | 0.25     | 0.138889 |\n",
      "+-------------+----------+-----------+----------+----------+\n",
      "| VOTING      | 0        | 0         | 0        | 0        |\n",
      "+-------------+----------+-----------+----------+----------+\n"
     ]
    }
   ],
   "source": [
    "from tabulate import tabulate\n",
    "\n",
    "# Assuming data is a 110x4 list, where each row is a sublist\n",
    "# data =  [[\"Row {} Col {}\".format(i + 1, j + 1) for j in range(4)] for i in range(110)]\n",
    "names_models = ['ADA',\n",
    "                'SVM',\n",
    "                'DNN',\n",
    "                'MLP',\n",
    "                'KNN',\n",
    "                'CAT',\n",
    "                'XGB',\n",
    "                'LGBM',\n",
    "                'RF',\n",
    "                'LR',\n",
    "                'VOTING',\n",
    "                'Bag_svm',\n",
    "                'Bag_knn',\n",
    "                'Bag_DT',\n",
    "                'Bag_LR',\n",
    "                'Bag_mlp',\n",
    "\n",
    "                'Bag_rf',\n",
    "                'Bag_ada',\n",
    "                'Bag_lgbm',\n",
    "                # 'Bag_xgb',\n",
    "                'Bag_cat',\n",
    "                'Bag_comb',\n",
    "                'avg',\n",
    "                'weighed_avg'\n",
    "                ]\n",
    "\n",
    "data = [[\"\" for _ in range(5)] for _ in range(len(names_models))]\n",
    "\n",
    "level_01_acc = [\n",
    "                ada_acc_01,\n",
    "                svm_acc_01,\n",
    "                dnn_acc_01,\n",
    "                mlp_acc_01,\n",
    "                knn_acc_01,\n",
    "                cat_acc_01,\n",
    "                xgb_acc_01,\n",
    "                lgbm_acc_01,\n",
    "                rf_acc_01,\n",
    "                lr_acc_01,\n",
    "                voting_acc_01,\n",
    "                bag_svm_acc_01,\n",
    "                bag_knn_acc_01,\n",
    "                bag_dt_acc_01,\n",
    "                bag_lr_acc_01,\n",
    "                bag_mlp_acc_01,\n",
    "\n",
    "                bag_rf_acc_01,\n",
    "                bag_ada_acc_01,\n",
    "                bag_lgbm_acc_01,\n",
    "                # bag_xgb_acc_01,\n",
    "                bag_cat_acc_01,\n",
    "                bag_comb_acc_01,\n",
    "\n",
    "                avg_acc_01,\n",
    "                weighed_avg_acc_01\n",
    "                ]  \n",
    "\n",
    "\n",
    "level_01_pre = [\n",
    "                ada_pre_01,\n",
    "                svm_pre_01,\n",
    "                dnn_pre_01,\n",
    "                mlp_pre_01,\n",
    "                knn_pre_01,\n",
    "                cat_pre_01,\n",
    "                xgb_pre_01,\n",
    "                lgbm_pre_01,\n",
    "                rf_pre_01,\n",
    "                lr_pre_01,\n",
    "                voting_pre_01,\n",
    "                bag_svm_pre_01,\n",
    "                bag_knn_pre_01,\n",
    "                bag_dt_pre_01,\n",
    "                bag_lr_pre_01,\n",
    "                bag_mlp_pre_01,\n",
    "\n",
    "                bag_rf_pre_01,\n",
    "                bag_ada_pre_01,\n",
    "                bag_lgbm_pre_01,\n",
    "                # bag_xgb_pre_01,\n",
    "                bag_cat_pre_01,\n",
    "                bag_comb_pre_01,\n",
    "\n",
    "                avg_pre_01,\n",
    "                weighed_avg_pre_01\n",
    "                ]  \n",
    "\n",
    "level_01_rec = [\n",
    "                ada_rec_01,\n",
    "                svm_rec_01,\n",
    "                dnn_rec_01,\n",
    "                mlp_rec_01,\n",
    "                knn_rec_01,\n",
    "                cat_rec_01,\n",
    "                xgb_rec_01,\n",
    "                lgbm_rec_01,\n",
    "                rf_rec_01,\n",
    "                lr_rec_01,\n",
    "                voting_rec_01,\n",
    "                bag_svm_rec_01,\n",
    "                bag_knn_rec_01,\n",
    "                bag_dt_rec_01,\n",
    "                bag_lr_rec_01,\n",
    "                bag_mlp_rec_01,\n",
    "\n",
    "                bag_rf_rec_01,\n",
    "                bag_ada_rec_01,\n",
    "                bag_lgbm_rec_01,\n",
    "                # bag_xgb_rec_01,\n",
    "                bag_cat_rec_01,\n",
    "                bag_comb_rec_01,\n",
    "\n",
    "                avg_rec_01,\n",
    "                weighed_avg_rec_01\n",
    "                ]  \n",
    "\n",
    "level_01_f1 = [\n",
    "                ada_f1_01,\n",
    "                svm_f1_01,\n",
    "                dnn_f1_01,\n",
    "                mlp_f1_01,\n",
    "                knn_f1_01,\n",
    "                cat_f1_01,\n",
    "                xgb_f1_01,\n",
    "                lgbm_f1_01,\n",
    "                rf_f1_01,\n",
    "                lr_f1_01,\n",
    "                voting_f1_01,\n",
    "                bag_svm_f1_01,\n",
    "                bag_knn_f1_01,\n",
    "                bag_dt_f1_01,\n",
    "                bag_lr_f1_01,\n",
    "                bag_mlp_f1_01,\n",
    "\n",
    "                bag_rf_f1_01,\n",
    "                bag_ada_f1_01,\n",
    "                bag_lgbm_f1_01,\n",
    "                # bag_xgb_f1_01,\n",
    "                bag_cat_f1_01,\n",
    "                bag_comb_f1_01,\n",
    "\n",
    "                avg_f1_01,\n",
    "                weighed_avg_f1_01\n",
    "                ]  \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Combine data into a list of tuples for sorting\n",
    "model_data = list(zip(names_models, level_01_acc, level_01_pre, level_01_rec, level_01_f1))\n",
    "\n",
    "# Sort by F1-01 score in descending order\n",
    "model_data_sorted = sorted(model_data, key=lambda x: x[4], reverse=True)\n",
    "\n",
    "# Separate the sorted data back into individual lists\n",
    "sorted_names_models, sorted_level_01_acc, sorted_level_01_pre, sorted_level_01_rec, sorted_level_01_f1 = zip(*model_data_sorted)\n",
    "\n",
    "# Assign the sorted data to the table\n",
    "for i in range(len(sorted_names_models)):\n",
    "    data[i][0] = sorted_names_models[i]\n",
    "    data[i][1] = sorted_level_01_acc[i]\n",
    "    data[i][2] = sorted_level_01_pre[i] \n",
    "    data[i][3] = sorted_level_01_rec[i] \n",
    "    data[i][4] = sorted_level_01_f1[i]\n",
    "\n",
    "# Define column headers\n",
    "headers = [\"Models\", \"ACC-01\", \"PRE-01\", \"REC-01\", \"F1-01\"]\n",
    "\n",
    "# Print the table\n",
    "table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "with open(output_file_name, \"a\") as f: print('Summary table', file = f)\n",
    "if pick_prob == 1: \n",
    "    with open(output_file_name, \"a\") as f: print('Level 01 - Probabilities', file = f)\n",
    "else:\n",
    "    with open(output_file_name, \"a\") as f: print('Level 01 - CLASSES', file = f)\n",
    "if feature_selection_bit == 1: \n",
    "    with open(output_file_name, \"a\") as f: print('Feature Selection was applied', file = f)\n",
    "else:\n",
    "    with open(output_file_name, \"a\") as f: print('All features were used', file = f)\n",
    "\n",
    "\n",
    "    \n",
    "print(table)\n",
    "with open(output_file_name, \"a\") as f: print(table, file = f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+----------------+\n",
      "| Models      |   time-01(sec) |\n",
      "+=============+================+\n",
      "| KNN         |      0.0286405 |\n",
      "+-------------+----------------+\n",
      "| Bag_DT      |      0.0345485 |\n",
      "+-------------+----------------+\n",
      "| LR          |      0.0382247 |\n",
      "+-------------+----------------+\n",
      "| Bag_knn     |      0.0421269 |\n",
      "+-------------+----------------+\n",
      "| Bag_svm     |      0.0472963 |\n",
      "+-------------+----------------+\n",
      "| avg         |      0.0580528 |\n",
      "+-------------+----------------+\n",
      "| SVM         |      0.0650797 |\n",
      "+-------------+----------------+\n",
      "| weighed_avg |      0.066397  |\n",
      "+-------------+----------------+\n",
      "| ADA         |      0.124664  |\n",
      "+-------------+----------------+\n",
      "| MLP         |      0.136247  |\n",
      "+-------------+----------------+\n",
      "| Bag_LR      |      0.139526  |\n",
      "+-------------+----------------+\n",
      "| CAT         |      0.182676  |\n",
      "+-------------+----------------+\n",
      "| RF          |      0.278346  |\n",
      "+-------------+----------------+\n",
      "| Bag_lgbm    |      0.300411  |\n",
      "+-------------+----------------+\n",
      "| Bag_ada     |      0.584041  |\n",
      "+-------------+----------------+\n",
      "| Bag_mlp     |      0.869736  |\n",
      "+-------------+----------------+\n",
      "| DNN         |      1.03695   |\n",
      "+-------------+----------------+\n",
      "| Bag_cat     |      1.54709   |\n",
      "+-------------+----------------+\n",
      "| Bag_rf      |      2.48274   |\n",
      "+-------------+----------------+\n",
      "| XGB         |     30.8813    |\n",
      "+-------------+----------------+\n",
      "| Bag_comb    |     31.7078    |\n",
      "+-------------+----------------+\n",
      "| LGBM        |     40.768     |\n",
      "+-------------+----------------+\n",
      "| VOTING      |   9999         |\n",
      "+-------------+----------------+\n"
     ]
    }
   ],
   "source": [
    "# implement time table\n",
    "from tabulate import tabulate\n",
    "\n",
    "names_models = ['ADA',\n",
    "                'SVM',\n",
    "                'DNN',\n",
    "                'MLP',\n",
    "                'KNN',\n",
    "                'CAT',\n",
    "                'XGB',\n",
    "                'LGBM',\n",
    "                'RF',\n",
    "                'LR',\n",
    "                'VOTING',\n",
    "                'Bag_svm',\n",
    "                'Bag_knn',\n",
    "                'Bag_DT',\n",
    "                'Bag_LR',\n",
    "                'Bag_mlp',\n",
    "\n",
    "                'Bag_rf',\n",
    "                'Bag_ada',\n",
    "                'Bag_lgbm',\n",
    "                # 'Bag_xgb',\n",
    "                'Bag_cat',\n",
    "                'Bag_comb',\n",
    "                'avg',\n",
    "                'weighed_avg'\n",
    "                ]\n",
    "\n",
    "data = [[\"\" for _ in range(2)] for _ in range(len(names_models))]\n",
    "\n",
    "level_01_time = [\n",
    "                ada_time_01,\n",
    "                svm_time_01,\n",
    "                dnn_time_01,\n",
    "                mlp_time_01,\n",
    "                knn_time_01,\n",
    "                cat_time_01,\n",
    "                xgb_time_01,\n",
    "                lgbm_time_01,\n",
    "                rf_time_01,\n",
    "                lr_time_01,\n",
    "                voting_time_01,\n",
    "                bag_svm_time_01,\n",
    "                bag_knn_time_01,\n",
    "                bag_dt_time_01,\n",
    "                bag_lr_time_01,\n",
    "                bag_mlp_time_01,\n",
    "\n",
    "                bag_rf_time_01,\n",
    "                bag_ada_time_01,\n",
    "                bag_lgbm_time_01,\n",
    "                # bag_xgb_time_01,\n",
    "                bag_cat_time_01,\n",
    "                bag_comb_time_01,\n",
    "\n",
    "                avg_time_01,\n",
    "                weighed_avg_time_01\n",
    "                ]  \n",
    "\n",
    "\n",
    "# Combine data into a list of tuples for sorting\n",
    "model_data = list(zip(names_models, level_01_time))\n",
    "\n",
    "# Sort by F1-01 score in descending order\n",
    "model_data_sorted = sorted(model_data, key=lambda x: x[1], reverse=False)\n",
    "\n",
    "# Separate the sorted data back into individual lists\n",
    "sorted_names_models, sorted_level_01_time = zip(*model_data_sorted)\n",
    "\n",
    "# Assign the sorted data to the table\n",
    "for i in range(len(sorted_names_models)):\n",
    "    data[i][0] = sorted_names_models[i]\n",
    "    data[i][1] = sorted_level_01_time[i]\n",
    "\n",
    "# Define column headers\n",
    "headers = [\"Models\", \"time-01(sec)\"]\n",
    "\n",
    "\n",
    "# Print the table\n",
    "table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "with open(output_file_name, \"a\") as f: print('Time is counted is seconds', file = f)\n",
    "print(table)\n",
    "with open(output_file_name, \"a\") as f: print(table, file = f)\n",
    "end_program = time.time()\n",
    "time_program = end_program - start_program\n",
    "with open(output_file_name, \"a\") as f: print('Running time of entire program is:', time_program ,' seconds',file = f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "if generate_feature_importance == 1:\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating SHAP explanation')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "  with open(output_file_name, \"a\") as f:print('ADA FEATURE IMPORTANCE',file = f)\n",
    "\n",
    "      #START TIMER MODEL\n",
    "  start = time.time()\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating explainer')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "  test = X_test_01\n",
    "  train = X_train_01\n",
    "  # ## Summary Bar Plot Global\n",
    "  start_index = 0\n",
    "  end_index = 250\n",
    "  # test.pop('Label')\n",
    "  # test.pop('is_train')\n",
    "  # print(label2)\n",
    "\n",
    "\n",
    "  # models = [ada,dnn_01,clf,knn_clf_01,cat_01,xgb_01, rf, lgbm, mlp,logreg_01]\n",
    "  explainer = shap.KernelExplainer(ada.predict_proba, test[start_index:end_index])\n",
    "\n",
    "  shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "\n",
    "  shap.summary_plot(shap_values = shap_values,\n",
    "                    features = test[start_index:end_index],\n",
    "                    # class_names=[column_features[:-1]],\n",
    "                    show=False)\n",
    "\n",
    "  # if feature_selection_bit == 1 # On\n",
    "  # pick_prob = 0 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "  if pick_prob == 1:\n",
    "    plt.savefig('ADA_SHAP_NSL_prob_01.png')\n",
    "  elif pick_prob == 0:\n",
    "    plt.savefig('ADA_SHAP_NSL_class_01.png')\n",
    "        \n",
    "  else: None\n",
    "  plt.clf()\n",
    "\n",
    "\n",
    "  vals= np.abs(shap_values).mean(1)\n",
    "  feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "  feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "  feature_importance.head()\n",
    "  print(feature_importance.to_string())\n",
    "\n",
    "  with open(output_file_name, \"a\") as f:print('Feature Importance: ',feature_importance.to_string(),file = f)\n",
    "\n",
    "\n",
    "\n",
    "  end = time.time()\n",
    "  with open(output_file_name, \"a\") as f:print('ELAPSE TIME LIME GLOBAL: ',(end - start)/60, 'min',file = f)\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  # feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "  feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "  # col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "  feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "  # for item1, item2 in zip(feature_name, feature_val):\n",
    "  #     print(item1, item2)\n",
    "\n",
    "\n",
    "  # Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "  zipped_lists = list(zip(feature_name, feature_val))\n",
    "  zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "  # Convert the sorted result back into separate lists\n",
    "  sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f: print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "  # with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "  # with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# explainer = shap.TreeExplainer(model)\n",
    "# start_index = 0\n",
    "# end_index = samples\n",
    "# shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "# shap_obj = explainer(test[start_index:end_index])\n",
    "# shap.summary_plot(shap_values = shap_values,\n",
    "#                   features = test[start_index:end_index],\n",
    "#                 show=False)\n",
    "# plt.savefig('Light_SHAP_CIC_Summary.png')\n",
    "# plt.clf()\n",
    "\n",
    "\n",
    "# vals= np.abs(shap_values).mean(1)\n",
    "# feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "# feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "# feature_importance.head()\n",
    "# print(feature_importance.to_string())\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "# # feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "# feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "# # col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "# feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "# # for item1, item2 in zip(feature_name, feature_val):\n",
    "# #     print(item1, item2)\n",
    "\n",
    "\n",
    "# # Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "# zipped_lists = list(zip(feature_name, feature_val))\n",
    "# zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "# # Convert the sorted result back into separate lists\n",
    "# sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "# for k in sorted_list1:\n",
    "#   with open(output_file_name, \"a\") as f:print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "# # with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "# for k in sorted_list1:\n",
    "#   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "# with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "# print('---------------------------------------------------------------------------------')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "if generate_feature_importance == 1:\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating SHAP explanation')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "\n",
    "  with open(output_file_name, \"a\") as f:print('XGB FEATURE IMPORTANCE',file = f)\n",
    "\n",
    "      #START TIMER MODEL\n",
    "  start = time.time()\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating explainer')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "  test = X_test_01\n",
    "  train = X_train_01\n",
    "  # ## Summary Bar Plot Global\n",
    "  start_index = 0\n",
    "  end_index = 250\n",
    "  # test.pop('Label')\n",
    "  # test.pop('is_train')\n",
    "  # print(label2)\n",
    "\n",
    "\n",
    "  explainer = shap.TreeExplainer(xgb_01)\n",
    "\n",
    "  shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "  shap_obj = explainer(test[start_index:end_index])\n",
    "  shap.summary_plot(shap_values = shap_values,\n",
    "                    features = test[start_index:end_index],\n",
    "                  show=False)\n",
    "  # plt.clf()\n",
    "\n",
    "  # if feature_selection_bit == 1 # On\n",
    "  # pick_prob = 0 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "  if pick_prob == 1:\n",
    "    plt.savefig('XGB_SHAP_NSL_prob_01.png')\n",
    "  elif pick_prob == 0:\n",
    "    plt.savefig('XGB_SHAP_NSL_class_01.png')\n",
    "\n",
    "  else: None\n",
    "  plt.clf()\n",
    "\n",
    "\n",
    "  vals= np.abs(shap_values).mean(1)\n",
    "  feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "  feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "  feature_importance.head()\n",
    "  print(feature_importance.to_string())\n",
    "  with open(output_file_name, \"a\") as f:print('Feature Importance: ',feature_importance.to_string(),file = f)\n",
    "\n",
    "\n",
    "\n",
    "  end = time.time()\n",
    "  with open(output_file_name, \"a\") as f:print('ELAPSE TIME LIME GLOBAL: ',(end - start)/60, 'min',file = f)\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  # feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "  feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "  # col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "  feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "  # for item1, item2 in zip(feature_name, feature_val):\n",
    "  #     print(item1, item2)\n",
    "\n",
    "\n",
    "  # Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "  zipped_lists = list(zip(feature_name, feature_val))\n",
    "  zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "  # Convert the sorted result back into separate lists\n",
    "  sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f: print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "  # with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "  # with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "if generate_feature_importance == 1:\n",
    "\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating SHAP explanation')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "\n",
    "  with open(output_file_name, \"a\") as f:print('LGBM FEATURE IMPORTANCE',file = f)\n",
    "\n",
    "      #START TIMER MODEL\n",
    "  start = time.time()\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating explainer')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "  test = X_test_01\n",
    "  train = X_train_01\n",
    "  # ## Summary Bar Plot Global\n",
    "  start_index = 0\n",
    "  end_index = 250\n",
    "  # test.pop('Label')\n",
    "  # test.pop('is_train')\n",
    "  # print(label2)\n",
    "\n",
    "\n",
    "  explainer = shap.TreeExplainer(lgbm)\n",
    "\n",
    "  shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "  shap_obj = explainer(test[start_index:end_index])\n",
    "  shap.summary_plot(shap_values = shap_values,\n",
    "                    features = test[start_index:end_index],\n",
    "                  show=False)\n",
    "  # plt.clf()\n",
    "\n",
    "  # if feature_selection_bit == 1 # On\n",
    "  # pick_prob = 0 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "  if pick_prob == 1:\n",
    "    plt.savefig('LGBM_SHAP_NSL_prob_01.png')\n",
    "  elif pick_prob == 0:\n",
    "    plt.savefig('LGBM_SHAP_NSL_class_01.png')\n",
    "\n",
    "  else: None\n",
    "  plt.clf()\n",
    "\n",
    "\n",
    "  vals= np.abs(shap_values).mean(1)\n",
    "  feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "  feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "  feature_importance.head()\n",
    "  print(feature_importance.to_string())\n",
    "  with open(output_file_name, \"a\") as f:print('Feature Importance: ',feature_importance.to_string(),file = f)\n",
    "\n",
    "\n",
    "\n",
    "  end = time.time()\n",
    "  with open(output_file_name, \"a\") as f:print('ELAPSE TIME LIME GLOBAL: ',(end - start)/60, 'min',file = f)\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  # feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "  feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "  # col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "  feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "  # for item1, item2 in zip(feature_name, feature_val):\n",
    "  #     print(item1, item2)\n",
    "\n",
    "\n",
    "  # Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "  zipped_lists = list(zip(feature_name, feature_val))\n",
    "  zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "  # Convert the sorted result back into separate lists\n",
    "  sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f: print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "  # with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "  # with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "if generate_feature_importance == 1:\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating SHAP explanation')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "\n",
    "  with open(output_file_name, \"a\") as f:print('RF FEATURE IMPORTANCE',file = f)\n",
    "\n",
    "      #START TIMER MODEL\n",
    "  start = time.time()\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('Generating explainer')\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  print('')\n",
    "  test = X_test_01\n",
    "  train = X_train_01\n",
    "  # ## Summary Bar Plot Global\n",
    "  start_index = 0\n",
    "  end_index = 250\n",
    "  # test.pop('Label')\n",
    "  # test.pop('is_train')\n",
    "  # print(label2)\n",
    "\n",
    "\n",
    "  explainer = shap.TreeExplainer(rf)\n",
    "\n",
    "  shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "  shap_obj = explainer(test[start_index:end_index])\n",
    "  shap.summary_plot(shap_values = shap_values,\n",
    "                    features = test[start_index:end_index],\n",
    "                  show=False)\n",
    "  # plt.clf()\n",
    "\n",
    "  # if feature_selection_bit == 1 # On\n",
    "  # pick_prob = 0 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "  if pick_prob == 1:\n",
    "    plt.savefig('RF_SHAP_NSL_prob_01.png')\n",
    "  elif pick_prob == 0:\n",
    "    plt.savefig('RF_SHAP_NSL_class_01.png')\n",
    "\n",
    "  else: None\n",
    "  plt.clf()\n",
    "\n",
    "\n",
    "  vals= np.abs(shap_values).mean(1)\n",
    "  feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "  feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "  feature_importance.head()\n",
    "  print(feature_importance.to_string())\n",
    "  with open(output_file_name, \"a\") as f:print('Feature Importance: ',feature_importance.to_string(),file = f)\n",
    "\n",
    "\n",
    "\n",
    "  end = time.time()\n",
    "  with open(output_file_name, \"a\") as f:print('ELAPSE TIME LIME GLOBAL: ',(end - start)/60, 'min',file = f)\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "  # feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "  feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "  # col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "  feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "  # for item1, item2 in zip(feature_name, feature_val):\n",
    "  #     print(item1, item2)\n",
    "\n",
    "\n",
    "  # Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "  zipped_lists = list(zip(feature_name, feature_val))\n",
    "  zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "  # Convert the sorted result back into separate lists\n",
    "  sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f: print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "  # with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "  # for k in sorted_list1:\n",
    "  #   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "  # with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "\n",
    "  print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
